improve memory usage reporting etc

ChangeLog

@@ -1,3 +1,19 @@
+2019-08-10 Richard Frith-Macdonald <rfm@gnu.org>
+
+	* EcMemoryLogger.h: Change protocol to pass more information.
+	* EcProcess.h: Update comments/documentation.
+	* EcProcess.m: Revise memory usage code to
+	Provide resident and data memory stats to the memory logger
+	Refrain from generating alerts (remove MemoryIncrement and
+	MemoryPercent settings) having a base memory calculated
+	simply as the average usage after ten minutes, plus 20 percent.
+	Generate alarms for memory usage between the base/allowed value
+	and the maximum allowed value.
+	Add MemoryIdle setting to specify an 'idle' hour during the day
+	in which the process will restart if it's near maximum allowed
+	memory (in the critical alarm range).
+	Change the status reporting to improve readability.
+
 2019-08-06 Richard Frith-Macdonald <rfm@gnu.org>
 
 	* EcCommand.m: Prevent repeated clears of alarms for newly connected

EcMemoryLogger.h

@@ -41,8 +41,13 @@
  * This callback is issued once per minute, with totalUsage representing
  * the memory usage of the process and notLeaked the amount of memory 
  * accounted for as active by -ecNotLeaked. All values are in bytes.
+ * The residentUsage reports the current resident (real memory rather
+ * than virtual memory) while the dataUsage is the amount of memory in
+ * the data segment (stack and heap).  
  */
-- (void)process: (EcProcess*)process
+- (void) process: (EcProcess*)process
     didUseMemory: (uint64_t)totalUsage
-      notLeaked: (uint64_t)notLeaked;
+       notLeaked: (uint64_t)notLeaked
+        resident: (uint64_t)residentUsage
+	    data: (uint64_t)dataUsage;
 @end

EcProcess.h

@@ -348,33 +348,24 @@ extern NSString*	cmdVersion(NSString *ver);
  *   <term>EcMemoryAllowed</term>
  *   <desc>
  *     This may be used to specify the process memory usage
- *     (in megabytes) before memory usage alerting may begin.<br />
+ *     (in megabytes) before memory usage alarms may begin.<br />
- *     Memory usage warning logs are then generated every time
- *     the average (over ten minutes) memory usage (adjusted by the
- *     average memory known not leaked) exceeds a warning
- *     threshold (the threshold is then increased).<br />
  *     If this setting is not specified (or a negative or excessive value
  *     is specified) then memory is monitored for ten minutes and
- *     the threshold is set at the peak during that period plus a
+ *     the threshold is set at the peak during that period plus a 20%
- *     margin to allow further memory growth before warning.<br />
+ *     margin to allow further memory growth.<br />
- *     The minimum margin is determined by the EcMemoryIncrement and
- *     EcMemoryPercentage settings.<br />
  *     This may be set in the NSUserDefaults system or in Control.plist,
  *     but may be overridden by using the 'memory' command in the
  *     Console program.
  *   </desc>
- *   <term>EcMemoryIncrement</term>
+ *   <term>EcMemoryIdle</term>
  *   <desc>
- *     This integer value controls the (KBytes) increment (from
+ *     This optional integer value (0 to 23) may be used to specify the hour
- *     current peak value) in process memory usage (adjusted by the
+ *     of the day in which restarts are preferred to take place (some time
- *     average memory known not leaked) after which
+ *     when the process is likely to be idle).  If a process is near a
- *     an alert is generated.<br />
+ *     maximum permitted memory usage at this time of day,
- *     If this is not set (or is set to a value less than 100KB or
+ *     the -ecRestart: method will be called.<br />
- *     greater than 1GB) then a value of 50MB is used.<br />
+ *     Near means over three quarters of the way between the base/allowed
- *     Setting a higher value makes memory leak detection less
+ *     memory usage (as determined by MemoryAllowed) and MemoryMaximum.<br />
- *     sensitive (but reduces unnecessary alerts).<br />
- *     If used in conjunction with EcMemoryPercentage, the greater
- *     of the two allowed memory values is used.<br />
  *     This may be set in the NSUserDefaults system or in Control.plist,
  *     but may be overridden by using the 'memory' command in the
  *     Console program.
@@ -382,29 +373,14 @@ extern NSString*	cmdVersion(NSString *ver);
  *   <term>EcMemoryMaximum</term>
  *   <desc>
  *     This may be used to specify the maximum process memory allowed
- *     (in megabytes) before the process is forced to quit due to
+ *     (in megabytes) before the process is forced to restart due to
  *     excessive memory usage.<br />
  *     If the total memory usage of the process reaches this threshold,
- *     the -cmdQuit: method will be called with an argument of -1.<br />
+ *     the -ecRestart: method will be called<br />
- *     If this is not specified (or a negative value is specified)
+ *     The process will also generate alarms depending on the memory usage
- *     the process will never shut down due to excessive memory usage.<br />
+ *     once the bas/allowed memory usge has been passed.  The severity of
- *     This may be set in the NSUserDefaults system or in Control.plist,
+ *     the alarms depends on how far from the base/allowed memory to the
- *     but may be overridden by using the 'memory' command in the
+ *     maximum memory the current ten minute average of the usage is.<br />
- *     Console program.
- *   </desc>
- *   <term>EcMemoryPercentage</term>
- *   <desc>
- *     This integer value controls the increase in the alerting
- *     threshold (adjusted by the average memory known not leaked)
- *     after which a memory usage alert is generated.<br />
- *     The increase is calculated as a percentage of the current
- *     peak memory usage value when an alert is generated.<br />
- *     If this is not set (or is set to a value less than 1 or
- *     greater than 100) then a value of 5 is used.<br />
- *     Setting a higher value make memory leak detection less
- *     sensitive (but reduces unnecessary alerts).<br />
- *     If used in conjunction with EcMemoryIncrement, the greater
- *     of the two allowed memory values is used.<br />
  *     This may be set in the NSUserDefaults system or in Control.plist,
  *     but may be overridden by using the 'memory' command in the
  *     Console program.
@@ -412,7 +388,7 @@ extern NSString*	cmdVersion(NSString *ver);
  *   <term>EcMemoryType</term>
  *   <desc>
  *    This controls the type of memory considered by the EcMemoryAllowed,
- *    EcMemoryIncrement, EcMemoryMaximum and EcMemoryPercentage options.<br />
+ *    EcMemoryIdle and EcMemoryMaximum options.<br />
  *    Total (the default), considers the total process memory usage.<br />
  *    Resident, considers current resident memory used.<br />
  *    Data, considers only dynamically allocated and stack memory.<br />
@@ -648,9 +624,9 @@ extern NSString*	cmdVersion(NSString *ver);
  */
 - (NSTimeInterval) ecQuitDuration;
 
-/** This method is designed for handling an orderly shutdown by calling
+/** This method is designed for handling an orderly shutdown by noting
- * -ecWillQuit: with the supplied reason, then -ecHandleQuit, and finally
+ * the supplied reason and status, and then calling -ecWillQuit,
- * calling -ecDidQuit: passing the supplied status.<br />
+ * -ecHandleQuit, and finally calling -ecDidQuit.<br />
  * Subclasses should not normally override this method. Instead override
  * the -ecHandleQuit method.<br />
  * For backward compatibility, this will call the -cmdQuit: method if a
@@ -665,11 +641,11 @@ extern NSString*	cmdVersion(NSString *ver);
  */
 - (NSTimeInterval) ecQuitLimit: (NSTimeInterval)seconds;
 
-/** Returns the quit reason supplied to the -ecQuitFor:with method.
+/** Returns the quit reason supplied to the -ecQuitFor:with: method.
  */
 - (NSString*) ecQuitReason;
 
-/** Returns the quit status supplied to the -ecQuitFor:with or -cmdQuit:
+/** Returns the quit status supplied to the -ecQuitFor:with: or -cmdQuit:
  * method.
  */
 - (NSInteger) ecQuitStatus;
@@ -680,7 +656,7 @@ extern NSString*	cmdVersion(NSString *ver);
 - (oneway void) ecReconnect;
 
 /** This method is designed for handling an orderly restart.<br />
- * The default implementation calls -ecQuitFor:status: with minus one as
+ * The default implementation calls -ecQuitFor:with: with minus one as
  * the status code so that the Command server will start the process
  * again.<br />
  * The method is called automatically when the MemoryMaximum limit is
@@ -911,9 +887,9 @@ extern NSString*	cmdVersion(NSString *ver);
  */
 - (NSMutableDictionary*)cmdOperator: (NSString*)name password: (NSString*)pass;
 
-/** This method calls -ecWillQuit: with a nil argument, then -ecHandleQuit,
+/** This method notes the supplied status and sets a nil quit reason, it
- * and finally calls -ecDidQuit: with the supplied value for the process
+ * then calls -ecWillQuit, -ecHandleQuit, and finally calls
- * exit status.<br />
+ * -ecDidQuit at process termination.<br />
  * Subclasses should override -ecHandleQuit rather than this method.
  */
 - (oneway void) cmdQuit: (NSInteger)status;
@@ -978,7 +954,7 @@ extern NSString*	cmdVersion(NSString *ver);
  * after any code dealing with the notifications has run.<br />
  * The return value of this method is used to control automatic generation
  * of alarms for fatal configuration errors by passing it to the
- * -ecConfigurationError: method.<br />
+ * -ecConfigurationError:,... method.<br />
  * When you implement this method, you must ensure that your implementation
  * calls the superclass implementation, and if that returns a non-nil
  * result, you should pass that on as the return value from your own
@@ -1295,7 +1271,8 @@ extern NSString*	cmdVersion(NSString *ver);
        perceivedSeverity: (EcAlarmSeverity)perceivedSeverity
 		 message: (NSString*)format, ... NS_FORMAT_FUNCTION(4,5);
 
-/** Supporting code called by the -ecException:message:... method.
+/** Supporting code called by the 
+ *  -ecException:specificProblem:perceivedSeverity:message:,... method.
  */
 - (EcAlarm*) ecException: (NSException*)cause
 	 specificProblem: (NSString*)specificProblem

EcProcess.m

@@ -777,13 +777,53 @@ static uint64_t	excPrev = 0;    // excluded usage at previous warning
 static uint64_t	memPrev = 0;    // total usage at previous warning
 static uint64_t	excPeak = 0;    // excluded peak usage
 static uint64_t	memPeak = 0;    // total peak usage
-static uint64_t	memWarn = 0;    // next warning interval
+static uint64_t	memBase = 0;    // base memory threshold
+static uint64_t	memWarn = 0;	// warning alarm threshold
+static uint64_t	memMinr = 0;	// minor alarm threshold
+static uint64_t	memMajr = 0;	// major alarm threshold
+static uint64_t	memCrit = 0;	// critical aarm threshold
 static uint64_t	memSlot = 0;    // minute counter
 static uint64_t	excRoll[10];    // last N values
 static uint64_t	memRoll[10];    // last N values
 #define	MEMCOUNT (sizeof(memRoll)/sizeof(*memRoll))
-static NSDate   *memTime = nil; // Time of last alert
 
+static void
+setMemBase()
+{
+  if (0 == memAllowed)
+    {
+      if (0 == memBase || memSlot < MEMCOUNT)
+	{
+	  memBase = (memAvge * 120) / 100;
+	}
+    }
+  else
+    {
+      memBase = memAllowed * 1024 * 1024;
+    }
+  if (memMaximum > 0)
+    {
+      uint64_t	max = memMaximum * 1024 * 1024;
+
+      if (max >= memBase + 1024)
+	{
+	  uint64_t	band = (max - memBase) / 4;
+
+	  memWarn = memBase;
+	  memMinr = memWarn + band;
+	  memMajr = memMinr + band;
+	  memCrit = memMajr + band;
+	}
+      else
+	{
+	  memWarn = memMinr = memMajr = memCrit = memBase;
+	}
+    }
+  else
+    {
+      memWarn = memMinr = memMajr = memCrit = 0;
+    }
+}
 
 static NSString*
 findAction(NSString *cmd)
@@ -2083,7 +2123,7 @@ static NSString	*noFiles = @"No log files to archive";
   if (memMaximum >= 4*1024)
     {
       [self cmdError: @"MemoryMaximum (%"PRIu64" too large for 32bit machine..."
-        @" using 0", memAllowed];
+        @" using 0", memMaximum];
       memMaximum = 0;	                // Disabled
     }
 #endif
@@ -4678,12 +4718,9 @@ With two parameters ('list' and a class),\n\
 With two parameters ('allowed' and a number),\n\
   the threshold for warnings about process size is set (in MB).\n\
   Set to 'default' to revert to the default.\n\
-With two parameters ('increment' and a number),\n\
+With two parameters ('idle' and a number),\n\
-  the size increment between warnings about process size is set (in KB\n\
+  the hour of the day when the process is considered idle is set.\n\
-  from 10 to 1048576).  Set to 'default' to revert to the default.\n\
+  Set to 'default' to revert to the default.\n\
-With two parameters ('percentage' and a number),\n\
-  the percentage increment between warnings about process memory size is\n\
-  set (from 1 to 1000).  Set to 'default' to revert to the default.\n\
 With two parameters ('maximum' and a number),\n\
   the maximum process size (in MB) is set.  On reaching the limit, the\n\
   process restarts unless the limit is zero (meaning no maximum).\n\
@@ -4788,37 +4825,26 @@ With two parameters ('maximum' and a number),\n\
                   [cmdDefs setCommand: arg forKey: @"MemoryAllowed"];
 		  [self cmdPrintf: @"MemoryAllowed set to %@MB.\n", arg];
                 }
-              memWarn = memAllowed * 1024 * 1024;
-              DESTROY(memTime);
               [self _memCheck];
             }
-          else if ([op caseInsensitiveCompare: @"increment"] == NSOrderedSame)
+          else if ([op caseInsensitiveCompare: @"idle"] == NSOrderedSame)
             {
-              if (val <= 100 || val > 1048576)
+	      if (!isdigit([arg characterAtIndex: 0]))
 		{
-                  [cmdDefs setCommand: nil forKey: @"MemoryIncrement"];
+		  val = -1;
-                  [self cmdPrintf: @"MemoryIncrement using default value.\n"];
+		}
+              if (val >= 0 && val < 24)
+                {
+                  arg = [NSString stringWithFormat: @"%d", (int)val];
+                  [cmdDefs setCommand: arg forKey: @"MemoryIdle"];
+		  [self cmdPrintf: @"MemoryIdle set to %@.\n", arg];
                 }
               else
                 {
-                  arg = [NSString stringWithFormat: @"%"PRIu64, (uint64_t)val];
+                  [cmdDefs setCommand: nil forKey: @"MemoryIdle"];
-                  [cmdDefs setCommand: arg forKey: @"MemoryIncrement"];
+		  [self cmdPrintf: @"MemoryIdle using default value.\n"];
-		  [self cmdPrintf: @"MemoryIncrement set to %@KB.\n", arg];
-                }
-            }
-          else if ([op caseInsensitiveCompare: @"percentage"] == NSOrderedSame)
-            {
-              if (val <= 0 || val > 1000)
-                {
-                  [cmdDefs setCommand: nil forKey: @"MemoryPercentage"];
-                  [self cmdPrintf: @"MemoryPercentage using default value.\n"];
-                }
-              else
-                {
-                  arg = [NSString stringWithFormat: @"%"PRIu64, (uint64_t)val];
-                  [cmdDefs setCommand: arg forKey: @"MemoryPercentage"];
-		  [self cmdPrintf: @"MemoryPercentage set to %@.\n", arg];
                 }
+              [self _memCheck];
             }
           else if ([op caseInsensitiveCompare: @"maximum"] == NSOrderedSame)
             {
@@ -4841,6 +4867,7 @@ With two parameters ('maximum' and a number),\n\
                   [cmdDefs setCommand: arg forKey: @"MemoryMaximum"];
 		  [self cmdPrintf: @"MemoryMaximum set to %@MB.\n", arg];
                 }
+              [self _memCheck];
             }
           else
             {
@@ -4953,30 +4980,51 @@ With two parameters ('maximum' and a number),\n\
 	    }
 	}
 
-      [self cmdPrintf: @"%@ memory usage: %"PRIu64"KB (current),"
+      [self cmdPrintf: @"%@ memory usage:\n  %"PRIu64"KB (current),"
         @" %"PRIu64"KB (peak)\n",
         memType, memLast/1024, memPeak/1024];
       [self cmdPrintf: @"  %"PRIu64"KB (average),"
         @" %"PRIu64"KB (start)\n",
         memAvge/1024, memStrt/1024];
-      [self cmdPrintf: @"              %"PRIu64"KB (reserved)\n",
+
-        excLast/1024];
+      setMemBase();
       if (memSlot < MEMCOUNT)
         {
-          [self cmdPrintf: @"Memory error reporting disabled (for %d min"
+          [self cmdPrintf: @"Waiting (for %d min"
-            @" of baseline stats collection).\n", (int)(MEMCOUNT - memSlot)];
+            @" of baseline stats collection).\n",
+	    (int)(MEMCOUNT - memSlot)];
         }
-      else
+
+      if (memAllowed > 0)
 	{
-          [self cmdPrintf:
+          [self cmdPrintf: @"MemoryAllowed: %"PRIu64
-            @"Memory error reporting after average usage: %"PRIu64"KB\n",
+	    @"KB\n  the process is expected to use up to this much memory.\n",
-            memWarn/1024];
+            memAllowed * 1024];
 	}
       if (memMaximum > 0)
         {
-          [self cmdPrintf:
+	  NSString	*idle;
-            @"Memory exceeded shutdown after peak usage: %"PRIu64"KB\n",
+	  int		hour;
-            memMaximum * 1024];
+
+	  if (0 == memAllowed)
+	    {
+	      [self cmdPrintf: @"Estimated base: %"PRIu64
+		@"KB\n  the process is expected to use up to"
+		@" this much memory.\n",
+		memAllowed * 1024];
+	    }
+          [self cmdPrintf: @"MemoryMaximum: %"PRIu64
+	    @"KB\n  the process is restarted when peak memory usage"
+	    @" is above this limit.\n", memMaximum * 1024];
+	  idle = [cmdDefs stringForKey: @"MemoryIdle"];
+	  if ([idle length] > 0 && (hour = [idle intValue]) >= 0 && hour < 24)
+	    {
+	      [self cmdPrintf: @"  The process is also restarted if memory"
+		@" is above %"PRIu64"KB\n  during the hour from %02d:00.\n",
+		memCrit/1024, hour];
+	    }
+	  [self cmdPrintf: @"Alarms are raised when memory usage"
+	    @" is above: %"PRIu64"KB.\n", memWarn / 1024];
         }
     }
 }
@@ -5149,10 +5197,9 @@ With two parameters ('maximum' and a number),\n\
 	    @"-%@HomeDirectory [relDir] Relative home within user directory\n"
 	    @"-%@UserDirectory [dir]    Override home directory for user\n"
 	    @"-%@Instance [aNumber]     Instance number for multiple copies\n"
-	    @"-%@MemoryAllowed [MB]     Expected memory usage (before alerts)\n"
+	    @"-%@MemoryAllowed [MB]     Expected memory usage (base size)\n"
-	    @"-%@MemoryIncrement [KB]   Absolute increase in alert threshold\n"
+	    @"-%@MemoryIdle [0-23]      Hour of day preferred for restart\n"
 	    @"-%@MemoryMaximum [MB]     Maximum memory usage (before restart)\n"
-	    @"-%@MemoryPercentage [N]   Percent increase in alert threshold\n"
 	    @"-%@MemoryType [aName]     Type of memory to measure. One of\n"
 	    @"                          Total, Resident or Data.\n"
 	    @"-%@ProgramName [aName]    Name to use for this program\n"
@@ -5571,7 +5618,13 @@ With two parameters ('maximum' and a number),\n\
 
 - (void) _memCheck
 {
+  static BOOL       	memRestart = NO;
+  static EcAlarm	*alarm = nil;
+  static char		buf[64] = {0};
+  EcAlarmSeverity	severity = EcAlarmSeverityCleared;
+  uint64_t  		mTotal, mResident, mShared, mText, mLib, mData, mDirty;
   BOOL                  memDebug = [cmdDefs boolForKey: @"Memory"];
+  int			pageSize = 4096;
   FILE          	*fptr;
   NSString		*str;
   int	        	i;
@@ -5600,13 +5653,16 @@ With two parameters ('maximum' and a number),\n\
 
   /* /proc/pid/statm reports the process memory size in 4KB pages
    */
-  fptr = fopen([[NSString stringWithFormat: @"/proc/%d/statm",
+  if ('\0' == *buf)
-    [[NSProcessInfo processInfo] processIdentifier]] UTF8String], "r");
+    {
+      sprintf(buf, "/proc/%d/statm",
+	[[NSProcessInfo processInfo] processIdentifier]);
+    }
+  fptr = fopen(buf, "r");
   memLast = 1;
+  mTotal = mResident = mShared = mText = mLib = mData = mDirty = 0;
   if (NULL != fptr)
     {
-      uint64_t  mTotal, mResident, mShared, mText, mLib, mData, mDirty;
-
       if (fscanf(fptr, "%"PRIu64" %"PRIu64" %"PRIu64" %"PRIu64" %"PRIu64
         " %"PRIu64" %"PRIu64,
         &mTotal, &mResident, &mShared, &mText, &mLib, &mData, &mDirty) != 7)
@@ -5627,7 +5683,7 @@ With two parameters ('maximum' and a number),\n\
 	    {
 	      memLast = mTotal;
 	    }
-          memLast *= (4 * 1024);
+          memLast *= pageSize;
           if (memLast <= 0) memLast = 1;
         }
       fclose(fptr);
@@ -5640,8 +5696,10 @@ With two parameters ('maximum' and a number),\n\
       NS_DURING
         {
           [cmdMemoryLogger process: self
-                      didUseMemory: memLast
+                      didUseMemory: mTotal * pageSize
-                         notLeaked: excLast];
+                         notLeaked: excLast
+			  resident: mResident * pageSize
+			      data: mData * pageSize];
         }
       NS_HANDLER
         {
@@ -5708,111 +5766,101 @@ With two parameters ('maximum' and a number),\n\
    */
   if (memMaximum > 0 && memPeak > (memMaximum * 1024 * 1024))
     {
-      static BOOL       memRestart = NO;
-
       if (NO == memRestart)
         {
           memRestart = YES;
-          [self cmdAlert: @"MemoryMaximum exceeded ... initiating restart"];
+          NSLog(@"MemoryMaximum exceeded ... initiating restart");
           [self ecRestart: @"memory usage limit reached"];
         }
       return;
     }
 
-  /* If the average memory usage is above the threshold (adjusted by any
+  setMemBase();
-   * change in known unleaked memory), we alert and reset the threshold.
+  if (memWarn > 0 && memAvge > memWarn)
-   * During the first ten minutes though, we always adjust the threshold and
-   * we suppress any warnings. This gives us a more stable starting point.
-   */
-  if (memAvge + excPrev - excAvge > memWarn || memSlot < MEMCOUNT)
     {
-      NSInteger     inc;
+      if (memAvge > memCrit)
-      NSInteger     pct;
-      uint64_t      iMax = 0;
-      uint64_t      pMax = 0;
-
-      /* We increase the threshold for the next alert by a percentage
-       * of the existing usage or by a fixed increment, whichever is
-       * the larger.
-       */
-      pct = [cmdDefs integerForKey: @"MemoryPercentage"];
-      if (pct < 1 || pct > 100)
 	{
-          /* Set the next alerting threshold 5%
+	  severity = EcAlarmSeverityCritical;
-           * the current peak usage,
-           * ensuring that only serious increases
-           * in usage will generate an alert.
-           */
-          pct = 5;
 	}
-      pMax = (memPeak * (100 + pct)) / 100;
+      else if (memAvge > memMajr)
-
-      inc = [cmdDefs integerForKey: @"MemoryIncrement"];
-      if (inc < 100 || inc > 1048576)
 	{
-          /* Set the next alerting threshold from
+	  severity = EcAlarmSeverityMajor;
-           * 50MB above the current peak usage,
-           * ensuring that only serious increases
-           * in usage will generate an alert.
-           */
-          inc = 50 * 1024;
 	}
-      iMax = memPeak + (inc * 1024);
+      else if (memAvge > memMinr)
-
-      memWarn = (iMax > pMax) ? iMax : pMax;
-      if (memWarn % 1024)
 	{
-          memWarn = (memWarn/1024 + 1) * 1024;
+	  severity = EcAlarmSeverityMinor;
-        }
-      if (memWarn < memAllowed * 1024 * 1024)
-        {
-          /* Never warn at less than the allowed memory.
-           */
-          memWarn = memAllowed * 1024 * 1024;
-        }
-
-      /* If not in the initial period, we need to generate an alert
-       * because the average has risen above the allowed size.
-       */
-      if (memSlot >= MEMCOUNT)
-        {
-          uint64_t      ePrev;
-          uint64_t      mPrev;
-          NSDate        *when;
-
-          ePrev = excPrev;
-          mPrev = memPrev;
-          when = AUTORELEASE(memTime);
-          excPrev = excAvge;
-          memPrev = memAvge;
-          memTime = [NSDate new];
-          if (nil == when)
-            {
-              [self cmdError: @"Average %@ memory usage %luKB (grown by %ldKB)"
-                @" with %luKB (grown by %ldKB) accounted for;"
-                @" possible leak of %ldKB (%u%%)",
-		memType,
-                (unsigned long)memAvge/1024,
-                (long)(memAvge - mPrev)/1024,
-                (unsigned long)excAvge/1024,
-                (long)(excAvge - ePrev)/1024,
-                (long)(memAvge - mPrev + ePrev - excAvge)/1024,
-                (unsigned)(((memAvge - mPrev + ePrev - excAvge)*100)/mPrev)];
 	}
       else
 	{
-              [self cmdError: @"Average %@ memory usage %luKB (grown by %ldKB)"
+	  severity = EcAlarmSeverityWarning;
-                @" with %luKB (grown by %ldKB) accounted for;"
+	}
-                @" possible leak of %ldKB (%u%%) since %@",
+    }
+
+  if (EcAlarmSeverityCleared == severity)
+    {
+      if (nil != alarm)
+	{
+	  EcAlarm	*clear = [alarm clear];
+
+	  DESTROY(alarm);
+	  [self alarm: clear];
+	}
+    }
+  else
+    {
+      NSString	*additional;
+
+      additional = [NSString stringWithFormat:
+	@"Average %@ memory usage %luKB (base %luKB, max %luKB)",
 	memType,
 	(unsigned long)memAvge/1024,
-                (long)(memAvge - mPrev)/1024,
+	(unsigned long)memBase/1024,
-                (unsigned long)excAvge/1024,
+	(unsigned long)memMaximum*1024];
-                (long)(excAvge - ePrev)/1024,
+
-                (long)(memAvge - mPrev + ePrev - excAvge)/1024,
+      NSLog(@"%@", additional);
-                (unsigned)(((memAvge - mPrev + ePrev - excAvge)*100)/mPrev),
+
-                when];
+      /* When we are critically close to reaching our memory limit
+       * AND it's the time of day when the process is idle, we need
+       * to restart.
+       */
+      if (EcAlarmSeverityCritical == severity)
+	{
+	  NSString	*idle;
+	  int		hour;
+
+	  /* Idle period is hour of the day and number of hours from 1 to 10
+	   */
+	  idle = [cmdDefs stringForKey: @"MemoryIdle"];
+	  if ([idle length] > 0 && (hour = [idle intValue]) >= 0 && hour < 24)
+	    {
+	      if ([[NSCalendarDate date] hourOfDay] == hour)
+		{
+		  if (NO == memRestart)
+		    {
+		      memRestart = YES;
+		      NSLog(@"MemoryMaximum near in idle time; restart");
+		      [self ecRestart: @"memory usage limit when idle"];
 		    }
+		  return;
+		}
+	    }
+	}
+
+      if ([alarm perceivedSeverity] != severity)
+	{
+	  EcAlarm	*a;
+
+	  a = [EcAlarm alarmForManagedObject: nil
+	    at: nil
+	    withEventType: EcAlarmEventTypeProcessingError
+	    probableCause: EcAlarmOutOfMemory
+	    specificProblem: @"MemoryAllowed exceeded"
+	    perceivedSeverity: severity
+	    proposedRepairAction: @"Investigate possible leak, monitor usage,"
+	      @" restart process when necessary."
+	    additionalText: additional];
+	  ASSIGN(alarm, a);
+	  [self alarm: alarm];
 	}
     }