Improve error handling when creating shared memory buffers. Don't use shared memory for windows that are (probably) image caches. Don't crash in -needsAlpha if we don't have valid data.

git-svn-id: svn+ssh://svn.gna.org/svn/gnustep/libs/back/trunk@14536 72102866-910b-0410-8b05-ffd578937521

ChangeLog

@@ -1,3 +1,11 @@
+2002-09-24 13:22  Alexander Malmberg  <alexander@malmberg.org>
+
+	* Source/x11/XWindowBuffer.m (+windowBufferForWindow:depthInfo:):
+	Detect and handle errors properly. Only use shared memory for
+	reasonably large windows.
+
+	(-needsAlpha): Make data isn't NULL before trying to do anything.
+
 2002-09-23 19:30  Alexander Malmberg  <alexander@malmberg.org>
 
 	* Source/art/ftfont.m (-initWithFontName:matrix:): Retain

Source/x11/XWindowBuffer.m

@@ -135,31 +135,68 @@ static int use_shape_hack = 0; /* this is an ugly hack :) */
 
       wi->ximage = NULL;
 
+      /* TODO: only use shared memory for 'real' on-screen windows. how can
+      we tell? don't create shared buffer until first expose? */
+      /* The primary problems seems to be the system limit on the _number_
+      of shared memory segments, not their total size, so we only create
+      shared buffers for reasonably large buffers and assume that the small
+      ones are just caches of images and will never be displayed, anyway
+      (and if they are displayed, it won't cost much, since they're small).
+      */
+      if (wi->window->xframe.size.width * wi->window->xframe.size.height < 4096)
+        goto no_xshm;
+
+#define WARN @" Falling back to normal XImage:s (will be slower)."
       /* Use XShm if possible, else fall back to normal XImage:s */
-      if (XShmQueryExtension(wi->display))
+      if (!XShmQueryExtension(wi->display))
 	{
+static BOOL xshm_warned = NO;
+	  if (!xshm_warned)
+	    NSLog(@"XShm not supported." WARN);
+	  xshm_warned = YES;
+	  goto no_xshm;
+	}
+
       wi->use_shm = 1;
       wi->ximage = XShmCreateImage(wi->display,
 	DefaultVisual(wi->display, DefaultScreen(wi->display)),
 	aDI->drawing_depth, ZPixmap, NULL, &wi->shminfo,
 	wi->window->xframe.size.width,
 	wi->window->xframe.size.height);
-	}
+      if (!wi->ximage)
-      if (wi->ximage)
 	{
-	  /* TODO: only use shared memory for 'real' on-screen
+	  NSLog(@"Warning: XShmCreateImage failed!" WARN);
-	     windows. how can we tell? */
+	  goto no_xshm;
+	}
       wi->shminfo.shmid = shmget(IPC_PRIVATE,
 	wi->ximage->bytes_per_line * wi->ximage->height,
 	IPC_CREAT | 0700);
 
-	  if (!wi->shminfo.shmid == -1)
+      if (wi->shminfo.shmid == -1)
-	    NSLog(@"shmget() failed"); /* TODO */
+	{
+	  NSLog(@"Warning: shmget() failed: %m." WARN);
+	  XDestroyImage(wi->ximage);
+	  goto no_xshm;
+	}
+
       wi->shminfo.shmaddr = wi->ximage->data = shmat(wi->shminfo.shmid, 0, 0);
+      if ((int)wi->shminfo.shmaddr == -1)
+	{
+	  NSLog(@"Warning: shmat() failed: %m." WARN);
+	  XDestroyImage(wi->ximage);
+	  shmctl(wi->shminfo.shmid, IPC_RMID, 0);
+	  goto no_xshm;
+	}
 
       wi->shminfo.readOnly = 0;
       if (!XShmAttach(wi->display, &wi->shminfo))
-	    NSLog(@"XShmAttach() failed");
+	{
+	  NSLog(@"Warning: XShmAttach() failed." WARN);
+	  XDestroyImage(wi->ximage);
+	  shmdt(wi->shminfo.shmaddr);
+	  shmctl(wi->shminfo.shmid, IPC_RMID, 0);
+	  goto no_xshm;
+	}
 
       /* We try to create a shared pixmap using the same buffer, and set
 	 it as the background of the window. This allows X to handle expose
@@ -172,25 +209,24 @@ static int use_shape_hack = 0; /* this is an ugly hack :) */
 				  wi->window->xframe.size.width,
 				  wi->window->xframe.size.height,
 				  aDI->drawing_depth);
-	  if (wi->pixmap)
+      if (wi->pixmap) /* TODO: this doesn't work */
 	{
 	  XSetWindowBackgroundPixmap(wi->display,wi->window->ident,wi->pixmap);
 	}
 
-	  /* On some systems (eg. freebsd), X can't attach to the shared
+      /* On some systems (eg. freebsd), X can't attach to the shared segment
-	  segment if it's marked for destruction, so we make sure it's
+      if it's marked for destruction, so we make sure it's attached before
-	  attached before marking it. */
+      marking it. */
       XSync(wi->display,False);
 
-	  /* Mark the segment as destroyed now. Since we're
+      /* Mark the segment as destroyed now. Since we're attached, it won't
-	     attached, it won't actually be destroyed, but if we
+      actually be destroyed, but if we crashed before doing this, it wouldn't
-	     crashed before doing this, it wouldn't be destroyed
+      be destroyed despite nobody being attached anymore. */
-	     despite nobody being attached anymore. */
       shmctl(wi->shminfo.shmid, IPC_RMID, 0);
-	}
+
-      else
+      if (!wi->ximage)
 	{
-	  NSLog(@"failed to create shared memory image, using normal XImage:s");
+no_xshm:
 	  wi->use_shm = 0;
 	  wi->ximage = XCreateImage(wi->display, DefaultVisual(wi->display,
 	    DefaultScreen(wi->display)), aDI->drawing_depth, ZPixmap, 0, NULL,
@@ -221,7 +257,7 @@ static int use_shape_hack = 0; /* this is an ugly hack :) */
     }
   else
     {
-      NSLog(@"Warning: failed to create image for window");
+      NSLog(@"Warning: failed to create image for window!");
       wi->data = NULL;
     }
 
@@ -420,6 +456,9 @@ static int warn = 0;
   if (has_alpha)
     return;
 
+  if (!data)
+    return;
+
 //	NSLog(@"needs alpha for %p: %ix%i", self, sx, sy);
 
   if (DI.inline_alpha)