Changes to NSMovie/NSMovieView/GSVideoSink and other classes to support playback of video files in NSMovie/NSMovieView

ChangeLog

@@ -82,6 +82,26 @@
 	* Source/NSImageCell.m: subclass initImageCell, so that
 	RefusesFirstResponder can be set, matching Mac.
 
+2022-03-27 Gregory John Casamento <greg.casamento@gmail.com>
+
+	* config.make.in: Add video flag
+	* configure.ac: Update to detect avformat.h
+	* Headers/Additions/GNUstepGUI/GSVideoSink.h
+	* Headers/Additions/GNUstepGUI/GSVideoSource.h: Add protocol
+        methods for reading and playing video.
+	* Headers/AppKit/NSMovie.h: Minor formatting changes.
+	* Headers/AppKit/NSMovieView.h: Minor formatting changes.
+	* Headers/AppKit/NSSound.h: Minor formatting changes
+	* Source/NSMovie.m: Fixes.
+	* Source/NSMovieView.m: Implement _stream method
+	* Source/NSSound.m: Minor formatting changes
+	* Tools/GNUmakefile: Cleanup
+	* Tools/video/GNUmakefile: Minor fixes
+	* Tools/video/VideoFileSource.m: Implementation of
+        readBytes:length: method for reading from the NSData.
+	* Tools/video/VideoOutputSink.m: Implementation of
+        playBytes:length: for playing the bytes as a movie.
+
 2022-02-26  Wolfgang Lux  <wolfgang.lux@gmail.com>
 
 	* Source/NSPopUpButtonCell.m(setMenu:): Select the first item of

Headers/Additions/GNUstepGUI/GSVideoSink.h

@@ -33,11 +33,12 @@
 #import <Foundation/NSByteOrder.h>
 #import <Foundation/NSObject.h>
 
+@class NSMovie;
+@class NSMovieView;
+
 @protocol GSVideoSink <NSObject>
 
-+ (BOOL) canInitWithData: (NSData *)data;
-
-/** 
+/**
  * Opens the device for output, called by [NSMovie-play].
  */
 - (BOOL) open;
@@ -60,6 +61,16 @@
  */
 - (CGFloat) volume;
 
+/**
+ * Sets the view to output to.
+ */
+- (void) setMovieView: (NSMovieView *)view;
+
+/**
+ * The movie view
+ */
+- (NSMovieView *) movieView;
+
 @end
 
 #endif // _GNUstep_H_GSVideoSink

Headers/AppKit/NSMovieView.h

@@ -33,6 +33,8 @@
 #import <AppKit/NSView.h>
 
 @class NSMovie;
+@class NSConditionLock;
+@class NSLock;
 
 typedef enum {
   NSQTMovieNormalPlayback,
@@ -44,9 +46,13 @@ APPKIT_EXPORT_CLASS
 @interface NSMovieView : NSView
 {
   @protected
-    NSMovie *_movie;
-    CGFloat  _rate;
-    CGFloat  _volume;
+    NSMovie         *_movie;
+    CGFloat          _rate;
+    CGFloat          _volume;
+    NSConditionLock *_readLock;
+    NSLock          *_playbackLock;
+    BOOL             _shouldLoop;
+    BOOL             _shouldStop;
     struct NSMovieViewFlags {
       unsigned int muted: 1;
       unsigned int loopMode: 3;

Headers/AppKit/NSSound.h

@@ -67,7 +67,7 @@ APPKIT_EXPORT_CLASS
   id<GSSoundSource> _source;
   id<GSSoundSink>   _sink;
   NSConditionLock   *_readLock;
-  NSLock            * _playbackLock;
+  NSLock            *_playbackLock;
   BOOL _shouldStop;
   BOOL _shouldLoop;
 }

Source/NSMovieView.m

@@ -28,28 +28,97 @@
 
 #import <Foundation/NSArray.h>
 #import <Foundation/NSData.h>
+#import <Foundation/NSLock.h>
 #import <Foundation/NSURL.h>
+
 #import "AppKit/NSMovie.h"
 #import "AppKit/NSMovieView.h"
 #import "AppKit/NSPasteboard.h"
 
+enum
+  {
+    MOVIE_SHOULD_PLAY = 1,
+    MOVIE_SHOULD_PAUSE
+  };
+
+#define BUFFER_SIZE 4096
+
 @interface NSMovie (NSMovieViewPrivate)
-- (id<GSVideoSource>) source;
-- (id<GSVideoSink>) sink;
+- (id<GSVideoSource>) _source;
+- (id<GSVideoSink>) _sink;
 @end
 
 @implementation NSMovie (NSMovieViewPrivate)
-- (id<GSVideoSource>) source
+- (id< GSVideoSource >) _source
 {
   return _source;
 }
 
-- (id<GSVideoSink>) sink
+- (id< GSVideoSink >) _sink
 {
   return _sink;
 }
 @end
 
+@interface NSMovieView (PrivateMethods)
+- (void) _stream;
+- (void) _finished: (NSNumber *)finishedPlaying;
+@end
+
+@implementation NSMovieView (PrivateMethods)
+- (void) _stream
+{
+  NSUInteger bytesRead;
+  BOOL success = NO;
+  void *buffer;
+  id <GSVideoSink> sink = [[self movie] _sink];
+  id <GSVideoSource> source = [[self movie] _source];
+  
+  // Exit with success = NO if device could not be open.
+  if ([sink open])
+    {
+      // Allocate space for buffer and start writing.
+      buffer = NSZoneMalloc(NSDefaultMallocZone(), BUFFER_SIZE);
+      do
+        {
+          do
+            {
+              // If not MOVIE_SHOULD_PLAY block thread
+              [_readLock lockWhenCondition: MOVIE_SHOULD_PLAY];
+              if (_shouldStop)
+                {
+                  [_readLock unlock];
+                  break;
+                }
+              bytesRead = [source readBytes: buffer
+                                     length: BUFFER_SIZE];
+              [_readLock unlock];
+              [_playbackLock lock];
+              success = [sink playBytes: buffer length: bytesRead];
+              [_playbackLock unlock];
+            } while ((!_shouldStop) && (bytesRead > 0) && success);
+          
+          [source setCurrentTime: 0.0];
+        } while (_shouldLoop == YES && _shouldStop == NO);
+      
+      [sink close];
+      NSZoneFree (NSDefaultMallocZone(), buffer);
+    }
+  
+  RETAIN(self);
+  [self performSelectorOnMainThread: @selector(_finished:)
+                         withObject: [NSNumber numberWithBool: success]
+                      waitUntilDone: YES];
+  RELEASE(self);
+}
+
+- (void) _finished: (NSNumber *)finishedPlaying
+{
+  DESTROY(_readLock);
+  DESTROY(_playbackLock);
+}
+@end
+
 @implementation NSMovieView
 
 - (instancetype) init
@@ -71,9 +140,15 @@
     }
   return self;
 }
-- (void) setMovie: (NSMovie*)movie
+
+- (void) setMovie: (NSMovie *)movie
 {
   ASSIGN(_movie, movie);
+
+  if (_movie != nil)
+    {
+      [[movie _sink] setMovieView: self];
+    }
 }
 
 - (NSMovie*) movie
@@ -83,18 +158,56 @@
 
 - (void) start: (id)sender
 {
-  //FIXME
+  // If the locks exists this instance is already playing
+  if (_readLock != nil && _playbackLock != nil)
+    {
+      return;
+    }
+  
+  _readLock = [[NSConditionLock alloc] initWithCondition: MOVIE_SHOULD_PAUSE];
+  _playbackLock = [[NSLock alloc] init];
+  
+  if ([_readLock tryLock] != YES)
+    {
+      return;
+    }
+  
+  _shouldStop = NO;
+  [NSThread detachNewThreadSelector: @selector(_stream)
+                           toTarget: self
+                         withObject: nil];
+
+  [_readLock unlockWithCondition: MOVIE_SHOULD_PLAY];
 }
 
 - (void) stop: (id)sender
 {
-  //FIXME
+    if (_readLock == nil)
+    {
+      return;
+    }
+  
+  if ([_readLock tryLock] != YES)
+    {
+      return;
+    }
+  _shouldStop = YES;
+
+  // Set to MOVIE_SHOULD_PLAY so that thread isn't blocked.
+  [_readLock unlockWithCondition: MOVIE_SHOULD_PLAY];
 }
 
 - (BOOL) isPlaying
 {
-  //FIXME
-  return NO;  
+  if (_readLock == nil)
+    {
+      return NO;
+    }
+  if ([_readLock condition] == MOVIE_SHOULD_PLAY)
+    {
+      return YES;
+    }
+  return NO;
 }
 
 - (void) gotoPosterFrame: (id)sender;

Source/NSSound.m

@@ -413,6 +413,7 @@ static inline void _loadNSSoundPlugIns (void)
       return NO;
     }
   _shouldStop = YES;
+
   // Set to SOUND_SHOULD_PLAY so that thread isn't blocked.
   [_readLock unlockWithCondition: SOUND_SHOULD_PLAY];
   

Tools/video/VideoFileSource.m

@@ -32,7 +32,7 @@
 
 #include <libavcodec/avcodec.h>
 
-#define INBUF_SIZE 4096
+#define INBUF_SIZE 4096 + AV_INPUT_BUFFER_PADDING_SIZE
 
 @interface VideofileSource : NSObject <GSVideoSource>
 {
@@ -70,14 +70,13 @@
 - (void)dealloc
 {
   TEST_RELEASE (_data);
-  // sf_close (_video);
-  
   [super dealloc];
 }
 
 - (id)initWithData: (NSData *)data
 {
   self = [super init];
+
   if (self == nil)
     {
       return nil;
@@ -91,7 +90,24 @@
 
 - (NSUInteger) readBytes: (void *)bytes length: (NSUInteger)length
 {
-  return 0; // (NSUInteger) (sf_read_short (_video, bytes, (length>>1))<<1);
+  NSRange range;
+  NSUInteger len = length - 1;
+
+  if (_curPos >= [_data length] - 1)
+    {
+      return 0;
+    }
+  
+  if (length > [_data length])
+    {
+      len = [_data length] - 1;
+    }
+  
+  range = NSMakeRange(_curPos, len);
+  [_data getBytes: bytes range: range];
+  _curPos += len;
+  
+  return len;
 }
 
 - (NSTimeInterval)duration
@@ -101,13 +117,10 @@
 
 - (void)setCurrentTime: (NSTimeInterval)currentTime
 {
-  // sf_count_t frames = (sf_count_t)((double)_info.samplerate * currentTime);
-  // sf_seek (_video, frames, SEEK_SET);
 }
+
 - (NSTimeInterval)currentTime
 {
-  // sf_count_t frames;
-  // frames = sf_seek (_video, 0, SEEK_CUR);
   return 0.0; // (NSTimeInterval)((double)frames / (double)_info.samplerate);
 }
 

Tools/video/VideoOutputSink.m

@@ -1,12 +1,12 @@
 /* 
    AudioOutputSink.m
 
-   Sink audio data to libao.
+   Sink audio data to libavcodec.
 
-   Copyright (C) 2009 Free Software Foundation, Inc.
+   Copyright (C) 2022 Free Software Foundation, Inc.
 
-   Written by:  Stefan Bidigaray <stefanbidi@gmail.com>
-   Date: Jun 2009
+   Written by:  Gregory John Casamento <greg.casamento@gmail.com>
+   Date: Mar 2022
    
    This file is part of the GNUstep GUI Library.
 
@@ -28,57 +28,250 @@
 */ 
 
 #include <Foundation/Foundation.h>
+#include <AppKit/NSMovie.h>
+#include <AppKit/NSMovieView.h>
+
 #include <GNUstepGUI/GSVideoSink.h>
 
+// Portions of this code have been taken from an example in avcodec by Fabrice Bellard
+
+/*
+ * Copyright (c) 2001 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <libavcodec/avcodec.h>
+
+#define INBUF_SIZE 4096
+
 @interface VideoOutputSink : NSObject <GSVideoSink>
 {
+  AVCodec *_codec;
+  AVCodecParserContext *_parser;
+  AVCodecContext *_context; // = NULL;
+  AVPacket *_pkt;
+  AVFrame *_frame;
+  
+  NSMovieView *_movieView;
 }
 @end
 
 @implementation VideoOutputSink
 
-+ (void) initialize
+- (void) display: (unsigned char *) buf
+            wrap: (int) wrap
+           xsize: (int) xsize
+           ysize: (int) ysize
 {
+  /*
+    FILE *f;
+    int i;
+
+    f = fopen(filename,"wb");
+    fprintf(f, "P5\n%d %d\n%d\n", xsize, ysize, 255);
+    for (i = 0; i < ysize; i++)
+        fwrite(buf + i * wrap, 1, xsize, f);
+    fclose(f);
+  */
 }
 
-+ (BOOL)canInitWithData: (NSData *)data
+
+
+- (void) decode
 {
-  return YES;
+  int ret;
+  
+  ret = avcodec_send_packet(_context, _pkt);
+  if (ret < 0)
+    {
+      NSLog(@"Error sending a packet for decoding\n");
+      return;
+    }
+  
+  while (ret >= 0)
+    {
+      ret = avcodec_receive_frame(_context, _frame);
+      if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
+        {
+          return;
+        }
+      else if (ret < 0)
+        {
+          NSLog(@"Error during decoding\n");
+          return;
+        }
+      
+      NSLog(@"saving frame %3d\n", _context->frame_number);
+      fflush(stdout);
+      
+      /* the picture is allocated by the decoder. no need to
+         free it */
+      // snprintf(buf, sizeof(buf), "%d", _context->frame_number);
+      [self display: _frame->data[0]
+               wrap: _frame->linesize[0]
+              xsize: _frame->width
+              ysize: _frame->height];
+      
+      // pgm_save(frame->data[0], frame->linesize[0],
+      //         frame->width, frame->height, buf);      
+    }
 }
 
 - (void)dealloc
 {
+  [self close];
+  _movieView = nil;
+  _pkt = NULL;
+  _context = NULL;
+  _parser = NULL;
+  _frame = NULL;
+  
   [super dealloc];
 }
 
 - (id)init
 {
   self = [super init];
+
   if (self != nil)
     {
+      _movieView = nil;
+      _pkt = NULL;
+      _context = NULL;
+      _parser = NULL;
+      _frame = NULL;
     }
+  
   return self;
 }
 
+- (void) setMovieView: (NSMovieView *)view
+{
+  _movieView = view; // weak, don't retain since the view is retained by its parent view.
+}
+
+- (NSMovieView *) movieView
+{
+  return _movieView;
+}
+
 - (BOOL)open
 {
+  _pkt = av_packet_alloc();
+  if (!_pkt)
+    {
+      NSLog(@"Could not allocate packet");
+      return NO;
+    }
+
+  _codec = avcodec_find_decoder(AV_CODEC_ID_MPEG4); // will set this based on file type.
+  if (!_codec)
+    {
+      NSLog(@"Could not find decoder for type");
+      return NO;
+    }
+  
+  _parser = av_parser_init(_codec->id);
+  if (!_parser)
+    {
+      NSLog(@"Could not init parser");
+      return NO;
+    }
+
+  _context = avcodec_alloc_context3(_codec);
+  if (!_context)
+    {
+      NSLog(@"Could not allocate video coder context");
+      return NO;
+    }
+  
+  /* open it */
+  if (avcodec_open2(_context, _codec, NULL) < 0)
+    {
+      NSLog(@"Could not open codec\n");
+      return NO;
+    }
+
+  _frame = av_frame_alloc();
+  if (!_frame)
+    {
+      NSLog(@"Could not allocate video frame\n");
+      return NO;
+    }
+  
   return YES;
 }
 
 - (void)close
 {
+  if (_parser != NULL)
+    {
+      av_parser_close(_parser);
+    }
+  
+  if (_context != NULL)
+    {
+      avcodec_free_context(&_context);
+    }
+  
+  if (_frame != NULL)
+    {
+      av_frame_free(&_frame);
+    }
+
+  if (_pkt != NULL)
+    {
+      av_packet_free(&_pkt);
+    }
 }
 
 - (BOOL)playBytes: (void *)bytes length: (NSUInteger)length
 {
+  int ret = av_parser_parse2(_parser, _context, &_pkt->data, &_pkt->size,
+                             bytes, length, AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0);
+  if (ret < 0)
+    {
+      NSLog(@"Error encountered while parsing data");
+      return NO;
+    }
+
+  if (_pkt != NULL)
+    {
+      if (_pkt->size)
+        {
+          [self decode];
+        }
+    }
+  else
+    {
+      return NO;
+    }
+  
+  return YES;
 }
 
-/* Functionality not supported by libao */
 - (void)setVolume: (float)volume
 {
 }
 
-- (float)volume
+- (CGFloat)volume
 {
   return 1.0;
 }

configure

@@ -5779,8 +5779,23 @@ fi
 
 done
 
-# Only if we have both...
-if test $have_video = yes -a $enable_video = yes; then
+for ac_header in libavformat/avformat.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "libavformat/avformat.h" "ac_cv_header_libavformat_avformat_h" "$ac_includes_default"
+if test "x$ac_cv_header_libavformat_avformat_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBAVFORMAT_AVFORMAT_H 1
+_ACEOF
+ have_format=yes
+else
+  have_format=no
+fi
+
+done
+
+
+# Only if we have both and enabled, then build video
+if test $have_video = yes -a $have_format -a $enable_video = yes; then
    BUILD_VIDEO="video"
 fi
 

configure.ac

@@ -571,8 +571,10 @@ BUILD_VIDEO=
 
 # Check for the headers...
 AC_CHECK_HEADERS(libavcodec/avcodec.h, have_video=yes, have_video=no)
-# Only if we have both...
-if test $have_video = yes -a $enable_video = yes; then
+AC_CHECK_HEADERS(libavformat/avformat.h, have_format=yes, have_format=no)
+
+# Only if we have both and enabled, then build video
+if test $have_video = yes -a $have_format -a $enable_video = yes; then
    BUILD_VIDEO="video"
 fi
 AC_SUBST(BUILD_VIDEO)