From e8d0a5efdcc795f5b91e7fe4f5ddc172346ef485 Mon Sep 17 00:00:00 2001 From: Richard Frith-MacDonald Date: Tue, 20 Aug 2013 14:35:26 +0000 Subject: [PATCH] memory usage handling improvments git-svn-id: svn+ssh://svn.gna.org/svn/gnustep/libs/ec/trunk@36993 72102866-910b-0410-8b05-ffd578937521 --- ChangeLog | 8 ++++ EcProcess.h | 41 ++++++++++++++++++-- EcProcess.m | 107 ++++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 129 insertions(+), 27 deletions(-) diff --git a/ChangeLog b/ChangeLog index 95b0aa3..b09ed6c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,11 @@ +2013-08-20 Richard Frith-Macdonald + + * EcConsole.m: Ignore exception printing message on shutdown. + * EcProcess.m: Improve SIGHUP handling. Add new option for + memory usage logging and increase default alert increments + again. Make name of allowed usage config consistent with other + names. Add option to quit process when usage gets too high. + 2013-08-05 Richard Frith-Macdonald * AlarmTool.m: Tool to raise/clear alarms diff --git a/EcProcess.h b/EcProcess.h index ce0303c..8c932ad 100644 --- a/EcProcess.h +++ b/EcProcess.h @@ -71,15 +71,50 @@ may be overridden by using the 'release' command in the Console program. + EcMemoryAllowed + + This may be used to specify the heap memory allocation allowed + (in megabytes) before memory usage alerting may begin.
+ If this is not specified (or a negative value is specified) + the default of 50 megabytes is used. + EcMemoryIncrement This integer value controls the (KBytes) increment in process memory usage after which an alert is generated.
- If this is not set (or is set to a value less than 1) then - a value of 500 is used unless EcMemory is set (in which case - the lower value of 20 is used).
+ If this is not set (or is set to a value less than ten or + greater than a million) then a value of five thousand is used + unless EcMemory is set (in which case twenty is used).
Setting a higher value make memory leak detection less sensitive (but reduces unnecessary alerts).
+ If used in conjunction with EcMemoryPercentage, the greater + of the two allowed memory values is used.
+ This may be set on the command line or in Control.plist + + EcMemoryMaximum + + This may be used to specify the heap memory allocation allowed + (in megabytes) before the process is forced to quit due to + excessive memory usage.
+ If the memory usage (heap usage minus -ecNotLeaked) reaches + this threshold, the -cmdQuit: method will be called with an + argument of -1.
+ If this is not specified (or a negative value is specified) + the process will never shut down due to excessive memory usage. + + EcMemoryPercentage + + This integer value controls the increase in the alerting + threshold after which a memory usage alert is generated.
+ The increase is calcuilated as a percentage of the current + memory usage value when an alert is generated.
+ If this is not set (or is set to a value less than one or + greater than a thousand) then a value of ten is used unless + EcMemory is set (in which one is used).
+ Setting a higher value make memory leak detection less + sensitive (but reduces unnecessary alerts).
+ If used in conjunction with EcMemoryIncrement, the greater + of the two allowed memory values is used.
This may be set on the command line or in Control.plist EcRelease diff --git a/EcProcess.m b/EcProcess.m index 0081306..032df2a 100644 --- a/EcProcess.m +++ b/EcProcess.m @@ -1,4 +1,3 @@ - /** Enterprise Control Configuration and Logging Copyright (C) 2012 Free Software Foundation, Inc. @@ -483,11 +482,12 @@ ecHostName() } #define DEFMEMALLOWED 50 -static int memAllowed = DEFMEMALLOWED; -static int memLast = 0; -static int memPeak = 0; -static int memSlot = 0; -static int memRoll[10]; +static uint64_t memMaximum = 0; +static uint64_t memAllowed = DEFMEMALLOWED; +static NSUInteger memLast = 0; +static NSUInteger memPeak = 0; +static NSUInteger memSlot = 0; +static NSUInteger memRoll[10]; #define memSize (sizeof(memRoll)/sizeof(*memRoll)) @@ -1103,14 +1103,16 @@ static NSString *noFiles = @"No log files to archive"; [self setCmdInterval: [str floatValue]]; } - str = [cmdDefs stringForKey: @"MemAllowed"]; - if (nil != str) + memAllowed = (uint64_t)[cmdDefs integerForKey: @"MemoryAllowed"]; + if (0 == memAllowed || memAllowed > 100000) { - memAllowed = [str intValue]; - if (memAllowed <= 0) - { - memAllowed = DEFMEMALLOWED; // Fifty megabytes default - } + memAllowed = DEFMEMALLOWED; // Fifty megabytes default + } + + memMaximum = (uint64_t)[cmdDefs integerForKey: @"MemoryMaximum"]; + if (memMaximum > 100000) + { + memMaximum = 0; // Disabled } str = [cmdDefs stringForKey: @"CoreSize"]; @@ -1895,6 +1897,7 @@ NSLog(@"Ignored attempt to set timer interval to %g ... using 10.0", interval); { NSLog(@"Rejected by Command - %@", [r objectForKey: @"rejected"]); + cmdIsQuitting = YES; [self cmdQuit: 0]; /* Rejected by server. */ } else if (nil == r || nil == [r objectForKey: @"back-off"]) @@ -1988,11 +1991,25 @@ NSLog(@"Ignored attempt to set timer interval to %g ... using 10.0", interval); struct mallinfo info; info = mallinfo(); + /* Do initial population so we can work immediately. + */ + if (0 == memRoll[0]) + { + int b = info.uordblks; + + if (b <= 0) b = 1; + while (memSlot < memSize) + { + memRoll[memSlot++] = b; + } + memLast = b; + memPeak = b; + } if (memSlot >= memSize) { NSUInteger average; NSUInteger notLeaked; - int i; + int i; notLeaked = [self ecNotLeaked]; @@ -2021,7 +2038,10 @@ NSLog(@"Ignored attempt to set timer interval to %g ... using 10.0", interval); average = ((average / 1024) + 1) * 1024; if (average > memPeak) { - int inc; + NSInteger inc; + NSInteger pct; + NSInteger iMax = 0; + NSInteger pMax = 0; /* Alert if the we have peaked above the allowed size. */ @@ -2044,8 +2064,28 @@ NSLog(@"Ignored attempt to set timer interval to %g ... using 10.0", interval); memPeak = size; } } - inc = (int)[cmdDefs integerForKey: @"MemoryIncrement"]; - if (inc < 1) + + /* If we have a defined maximum memory usage for the process, + * we should shut down with a non-zero status to get a restart. + */ + if (memMaximum > 0 && memPeak > memMaximum * 1024 * 1024) + { + if (NO == cmdIsQuitting) + { + cmdIsQuitting = YES; + [self cmdQuit: -1]; + } + } + + /* We increase the threshold for the next alert by a percentage + * of the existing usage or by a fixed increment, whichever is + * the larger. + */ + pct = [cmdDefs integerForKey: @"MemoryPercentage"]; + if (pct < 1 || pct > 1000) pct = 0; + inc = [cmdDefs integerForKey: @"MemoryIncrement"]; + if (inc < 10 || inc > 1000000) inc = 0; + if (0 == inc && 0 == pct) { if (YES == [cmdDefs boolForKey: @"Memory"]) { @@ -2054,6 +2094,7 @@ NSLog(@"Ignored attempt to set timer interval to %g ... using 10.0", interval); * peak usage. */ inc = 20; + pct = 0; } else { @@ -2063,10 +2104,19 @@ NSLog(@"Ignored attempt to set timer interval to %g ... using 10.0", interval); * ensuring that only serious increases * in usage will generate an alert. */ - inc = 5120; + inc = 5000; + pct = 10; // Use ten percent if more than fixed increment } } - memPeak = ((memPeak / (inc * 1024)) + 2) * (inc * 1024); + if (inc > 0) + { + iMax = ((memPeak / (inc * 1024)) + 2) * (inc * 1024); + } + if (pct > 0) + { + pMax = (((memPeak * (100 + pct)) / 1024 + 1) * 1024); + } + memPeak = (iMax > pMax) ? iMax : pMax; } } /* Record the latest memory usage. @@ -2161,6 +2211,7 @@ NSLog(@"Ignored attempt to set timer interval to %g ... using 10.0", interval); int sig = cmdSignalled; cmdSignalled = 0; + cmdIsQuitting = YES; [self cmdQuit: sig]; } } @@ -2176,6 +2227,7 @@ NSLog(@"Ignored attempt to set timer interval to %g ... using 10.0", interval); /* finish server */ + cmdIsQuitting = YES; [self cmdQuit: 0]; DESTROY(EcProcConnection); return 0; @@ -2970,11 +3022,16 @@ NSLog(@"Ignored attempt to set timer interval to %g ... using 10.0", interval); } } - if (YES == [cmdDefs boolForKey: @"Memory"]) - { - [self cmdPrintf: @"Memory usage: %u (peak), %u (current)\n", - memPeak, memLast]; - } + [self cmdPrintf: @"Memory usage: %"PRIuPTR" (peak), %"PRIuPTR + " (current), %"PRIuPTR" (exempt)\n", + memPeak, memLast, [self ecNotLeaked]]; + [self cmdPrintf: + @"Memory error reporting after usage: %"PRIu64"MB\n", memAllowed]; + if (memMaximum > 0) + { + [self cmdPrintf: + @"Memory usage exceeded shutdown after: %"PRIu64"MB\n", memMaximum]; + } } } @@ -3869,6 +3926,7 @@ NSLog(@"Ignored attempt to set timer interval to %g ... using 10.0", interval); cmdPTimer = nil; if (sig > 0) { + cmdIsQuitting = YES; [self cmdQuit: sig]; } if (YES == inProgress) @@ -4071,6 +4129,7 @@ NSLog(@"Ignored attempt to set timer interval to %g ... using 10.0", interval); additionalText: err]; [self alarm: a]; [alarmDestination shutdown]; + cmdIsQuitting = YES; [self cmdQuit: 1]; } else