[关闭]
@qidiandasheng 2020-07-13T14:45:18.000000Z 字数 12513 阅读 1704

GPUImage解析（一）：GPUImageOutput

音视频

GPUImageOutput

GPUImage响应链条上最先需要有输入源输入图片数据，这些输入源主要继承自GPUImageOutput。我们常用到的主要是以下几个：

这里我们将以GPUImageVideoCamera为例来解析一下响应链上的源数据的一个处理。

GPUImageVideoCamera

回调函数获取图像数据

GPUImageVideoCamera主要使用AVCaptureSession类从AV输入设备的采集数据到制定的输出。关于AVCaptureSession的使用这里就不讲了，主要说一下对应的采集数据输出回调函数之后的处理。

以下方法为AVCaptureVideoDataOutput的输出回调函数，省去部分代码，我们主要看处理图像部分的代码：

  1. - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection{
  3.         // 帧渲染的信号量，用于等待处理完一帧后，再接着处理下一帧
  4.         // 初始信号为1，调用wait后为0，继续往下执行任务加入队列
  5.         if (dispatch_semaphore_wait(frameRenderingSemaphore, DISPATCH_TIME_NOW) != 0)
  6.         {
  7.             return;
  8.         }
  10.         CFRetain(sampleBuffer);
  11.         // 把获取到的每一帧图像放入到异步队列中进行处理
  12.         runAsynchronouslyOnVideoProcessingQueue(^{
  13.             // 回调（调用方可获取摄像头捕获的回调数据CMSampleBufferRef）
  14.             if (self.delegate)
  15.             {
  16.                 [self.delegate willOutputSampleBuffer:sampleBuffer];
  17.             }
  19.             // 核心处理图像数据的代码
  20.             [self processVideoSampleBuffer:sampleBuffer];
  22.             CFRelease(sampleBuffer);
  23.             dispatch_semaphore_signal(frameRenderingSemaphore);
  24.         });
  25. }

图像处理

  1. - (void)processVideoSampleBuffer:(CMSampleBufferRef)sampleBuffer;
  2. {
  3.     if (capturePaused)
  4.     {
  5.         return;
  6.     }
  8.     //获取当前时间
  9.     CFAbsoluteTime startTime = CFAbsoluteTimeGetCurrent();
  10.     //获取当前视频帧
  11.     CVImageBufferRef cameraFrame = CMSampleBufferGetImageBuffer(sampleBuffer);
  13.     //获取视频的宽高
  14.     int bufferWidth = (int) CVPixelBufferGetWidth(cameraFrame);
  15.     int bufferHeight = (int) CVPixelBufferGetHeight(cameraFrame);
  16.     //获取颜色空间格式
  17.     CFTypeRef colorAttachments = CVBufferGetAttachment(cameraFrame, kCVImageBufferYCbCrMatrixKey, NULL);
  19.     //选择对应的颜色转换矩阵（YUV颜色空间到RGB颜色空间的转换矩阵）
  20.     if (colorAttachments != NULL)
  21.     {
  22.         if(CFStringCompare(colorAttachments, kCVImageBufferYCbCrMatrix_ITU_R_601_4, 0) == kCFCompareEqualTo)
  23.         {
  24.             if (isFullYUVRange)
  25.             {
  26.                 _preferredConversion = kColorConversion601FullRange;
  27.             }
  28.             else
  29.             {
  30.                 _preferredConversion = kColorConversion601;
  31.             }
  32.         }
  33.         else
  34.         {
  35.             _preferredConversion = kColorConversion709;
  36.         }
  37.     }
  38.     else
  39.     {
  40.         if (isFullYUVRange)
  41.         {
  42.             _preferredConversion = kColorConversion601FullRange;
  43.         }
  44.         else
  45.         {
  46.             _preferredConversion = kColorConversion601;
  47.         }
  48.     }
  50.     //获取样本时间戳
  51.     CMTime currentTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
  53.     //创建上下文EAGLContext，并设置为当前上下文环境
  54.     [GPUImageContext useImageProcessingContext];
  56.     /*是否支持快速读取CVPixelBufferRef，是否是YUV格式
  57.     模拟器不支持
  58.     真机iOS5开始支持的一种CVOpenGLESTextureCacheRef和CVImageBufferRef的映射
  59.      */
  60.     if ([GPUImageContext supportsFastTextureUpload] && captureAsYUV)
  61.     {
  62.         //CV纹理的缓存（亮度纹理和色度纹理）
  63.         CVOpenGLESTextureRef luminanceTextureRef = NULL;
  64.         CVOpenGLESTextureRef chrominanceTextureRef = NULL;
  66.         //得到YUV像素缓冲区平面数量
  67.         if (CVPixelBufferGetPlaneCount(cameraFrame) > 0) // Check for YUV planar inputs to do RGB conversion
  68.         {
  69.             //锁定缓存
  70.             CVPixelBufferLockBaseAddress(cameraFrame, 0);
  72.             //根据CVImageBufferRef纹理得到的宽高赋值
  73.             if ( (imageBufferWidth != bufferWidth) && (imageBufferHeight != bufferHeight) )
  74.             {
  75.                 imageBufferWidth = bufferWidth;
  76.                 imageBufferHeight = bufferHeight;
  77.             }
  79.             CVReturn err;
  80.             // Y-plane
  81.             //设置纹理单元，为了给着色器程序绑定纹理ID
  82.             glActiveTexture(GL_TEXTURE4);
  83.             //判断本设备是否支持红色的纹理，创建亮度纹理引用对象
  84.             if ([GPUImageContext deviceSupportsRedTextures])
  85.             {
  86.                 //将BGRA的数据转换到RGBA的纹理上
  87.                 err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, [[GPUImageContext sharedImageProcessingContext] coreVideoTextureCache], cameraFrame, NULL, GL_TEXTURE_2D, GL_LUMINANCE, bufferWidth, bufferHeight, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &luminanceTextureRef);
  88.             }
  89.             else
  90.             {
  91.                 err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, [[GPUImageContext sharedImageProcessingContext] coreVideoTextureCache], cameraFrame, NULL, GL_TEXTURE_2D, GL_LUMINANCE, bufferWidth, bufferHeight, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &luminanceTextureRef);
  92.             }
  93.             if (err)
  94.             {
  95.                 NSLog(@"Error at CVOpenGLESTextureCacheCreateTextureFromImage %d", err);
  96.             }
  98.             // 获取亮度纹理对象
  99.             luminanceTexture = CVOpenGLESTextureGetName(luminanceTextureRef);
  100.             // 绑定纹理
  101.             glBindTexture(GL_TEXTURE_2D, luminanceTexture);
  102.             // 设置纹理滤波
  103.             glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
  104.             glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
  106.             // UV-plane
  107.             glActiveTexture(GL_TEXTURE5);
  108.             //判断本设备是否支持红色的纹理，创建色度纹理引用对象
  109.             if ([GPUImageContext deviceSupportsRedTextures])
  110.             {
  111.                 err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, [[GPUImageContext sharedImageProcessingContext] coreVideoTextureCache], cameraFrame, NULL, GL_TEXTURE_2D, GL_LUMINANCE_ALPHA, bufferWidth/2, bufferHeight/2, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, 1, &chrominanceTextureRef);
  112.             }
  113.             else
  114.             {
  115.                 err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, [[GPUImageContext sharedImageProcessingContext] coreVideoTextureCache], cameraFrame, NULL, GL_TEXTURE_2D, GL_LUMINANCE_ALPHA, bufferWidth/2, bufferHeight/2, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, 1, &chrominanceTextureRef);
  116.             }
  117.             if (err)
  118.             {
  119.                 NSLog(@"Error at CVOpenGLESTextureCacheCreateTextureFromImage %d", err);
  120.             }
  122.             // 获取色度纹理对象
  123.             chrominanceTexture = CVOpenGLESTextureGetName(chrominanceTextureRef);
  124.             glBindTexture(GL_TEXTURE_2D, chrominanceTexture);
  125.             glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
  126.             glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
  128.             //YUV转换成RGB输出
  129.             [self convertYUVToRGBOutput];
  131.             //旋转后的宽高
  132.             int rotatedImageBufferWidth = bufferWidth, rotatedImageBufferHeight = bufferHeight;
  134.             if (GPUImageRotationSwapsWidthAndHeight(internalRotation))
  135.             {
  136.                 rotatedImageBufferWidth = bufferHeight;
  137.                 rotatedImageBufferHeight = bufferWidth;
  138.             }
  140.             //通过纹理缓存更新target
  141.             [self updateTargetsForVideoCameraUsingCacheTextureAtWidth:rotatedImageBufferWidth height:rotatedImageBufferHeight time:currentTime];
  143.             CVPixelBufferUnlockBaseAddress(cameraFrame, 0);
  144.             CFRelease(luminanceTextureRef);
  145.             CFRelease(chrominanceTextureRef);
  146.         }
  147.         else
  148.         {
  150.         }
  153.         if (_runBenchmark)
  154.         {
  155.             numberOfFramesCaptured++;
  156.             if (numberOfFramesCaptured > INITIALFRAMESTOIGNOREFORBENCHMARK)
  157.             {
  158.                 CFAbsoluteTime currentFrameTime = (CFAbsoluteTimeGetCurrent() - startTime);
  159.                 totalFrameTimeDuringCapture += currentFrameTime;
  160.                 NSLog(@"Average frame time : %f ms", [self averageFrameDurationDuringCapture]);
  161.                 NSLog(@"Current frame time : %f ms", 1000.0 * currentFrameTime);
  162.             }
  163.         }
  164.     }
  165.     else
  166.     {
  167.         CVPixelBufferLockBaseAddress(cameraFrame, 0);
  169.         int bytesPerRow = (int) CVPixelBufferGetBytesPerRow(cameraFrame);
  170.         outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:CGSizeMake(bytesPerRow / 4, bufferHeight) onlyTexture:YES];
  171.         [outputFramebuffer activateFramebuffer];
  173.         glBindTexture(GL_TEXTURE_2D, [outputFramebuffer texture]);
  175.         //        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, bufferWidth, bufferHeight, 0, GL_BGRA, GL_UNSIGNED_BYTE, CVPixelBufferGetBaseAddress(cameraFrame));
  177.         // Using BGRA extension to pull in video frame data directly
  178.         // The use of bytesPerRow / 4 accounts for a display glitch present in preview video frames when using the photo preset on the camera
  179.         glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, bytesPerRow / 4, bufferHeight, 0, GL_BGRA, GL_UNSIGNED_BYTE, CVPixelBufferGetBaseAddress(cameraFrame));
  181.         [self updateTargetsForVideoCameraUsingCacheTextureAtWidth:bytesPerRow / 4 height:bufferHeight time:currentTime];
  183.         CVPixelBufferUnlockBaseAddress(cameraFrame, 0);
  185.         if (_runBenchmark)
  186.         {
  187.             numberOfFramesCaptured++;
  188.             if (numberOfFramesCaptured > INITIALFRAMESTOIGNOREFORBENCHMARK)
  189.             {
  190.                 CFAbsoluteTime currentFrameTime = (CFAbsoluteTimeGetCurrent() - startTime);
  191.                 totalFrameTimeDuringCapture += currentFrameTime;
  192.             }
  193.         }
  194.     }  
  195. }
  1. // YUV转换成RGB输出
  2. - (void)convertYUVToRGBOutput;
  3. {
  4.     // 设置顶点着色器
  5.     [GPUImageContext setActiveShaderProgram:yuvConversionProgram];
  7.     int rotatedImageBufferWidth = imageBufferWidth, rotatedImageBufferHeight = imageBufferHeight;
  9.     if (GPUImageRotationSwapsWidthAndHeight(internalRotation))
  10.     {
  11.         rotatedImageBufferWidth = imageBufferHeight;
  12.         rotatedImageBufferHeight = imageBufferWidth;
  13.     }
  15.     // 从当前上下文中获取输出帧缓冲
  16.     outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:CGSizeMake(rotatedImageBufferWidth, rotatedImageBufferHeight) textureOptions:self.outputTextureOptions onlyTexture:NO];
  17.     [outputFramebuffer activateFramebuffer];
  19.     glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
  20.     glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
  22.     static const GLfloat squareVertices[] = {
  23.         -1.0f, -1.0f,
  24.         1.0f, -1.0f,
  25.         -1.0f,  1.0f,
  26.         1.0f,  1.0f,
  27.     };
  29.     // 绑定亮度纹理到纹理单元4
  30.     glActiveTexture(GL_TEXTURE4);
  31.     glBindTexture(GL_TEXTURE_2D, luminanceTexture);
  32.     // 把位置4的纹理单元赋值给采样器
  33.     glUniform1i(yuvConversionLuminanceTextureUniform, 4);
  35.     // 绑定色度纹理到纹理单元5
  36.     glActiveTexture(GL_TEXTURE5);
  37.     glBindTexture(GL_TEXTURE_2D, chrominanceTexture);
  38.     // 把位置5的纹理单元赋值给采样器
  39.     glUniform1i(yuvConversionChrominanceTextureUniform, 5);
  41.     glUniformMatrix3fv(yuvConversionMatrixUniform, 1, GL_FALSE, _preferredConversion);
  43.     //设置顶点数据
  44.     glVertexAttribPointer(yuvConversionPositionAttribute, 2, GL_FLOAT, 0, 0, squareVertices);
  45.     //设置纹理数据
  46.     glVertexAttribPointer(yuvConversionTextureCoordinateAttribute, 2, GL_FLOAT, 0, 0, [GPUImageFilter textureCoordinatesForRotation:internalRotation]);
  48.     //开始绘制
  49.     glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
  50. }

更新targets的输入帧缓冲并通知渲染

以下代码遍历了两次当前所有的targets。

第一次给target设置各种参数：旋转、大小、帧缓冲（绑定的附件为纹理缓冲）。

第二次通知所有的target开始重新绘制渲染纹理。

  1. - (void)updateTargetsForVideoCameraUsingCacheTextureAtWidth:(int)bufferWidth height:(int)bufferHeight time:(CMTime)currentTime;
  2. {
  3.     // First, update all the framebuffers in the targets
  4.     for (id<GPUImageInput> currentTarget in targets)
  5.     {
  6.         if ([currentTarget enabled])
  7.         {
  8.             NSInteger indexOfObject = [targets indexOfObject:currentTarget];
  9.             NSInteger textureIndexOfTarget = [[targetTextureIndices objectAtIndex:indexOfObject] integerValue];
  11.             if (currentTarget != self.targetToIgnoreForUpdates)
  12.             {
  13.                 [currentTarget setInputRotation:outputRotation atIndex:textureIndexOfTarget];
  14.                 [currentTarget setInputSize:CGSizeMake(bufferWidth, bufferHeight) atIndex:textureIndexOfTarget];
  16.                 if ([currentTarget wantsMonochromeInput] && captureAsYUV)
  17.                 {
  18.                     [currentTarget setCurrentlyReceivingMonochromeInput:YES];
  19.                     // TODO: Replace optimization for monochrome output
  20.                     [currentTarget setInputFramebuffer:outputFramebuffer atIndex:textureIndexOfTarget];
  21.                 }
  22.                 else
  23.                 {
  24.                     [currentTarget setCurrentlyReceivingMonochromeInput:NO];
  25.                     [currentTarget setInputFramebuffer:outputFramebuffer atIndex:textureIndexOfTarget];
  26.                 }
  27.             }
  28.             else
  29.             {
  30.                 [currentTarget setInputRotation:outputRotation atIndex:textureIndexOfTarget];
  31.                 [currentTarget setInputFramebuffer:outputFramebuffer atIndex:textureIndexOfTarget];
  32.             }
  33.         }
  34.     }
  36.     // Then release our hold on the local framebuffer to send it back to the cache as soon as it's no longer needed
  37.     [outputFramebuffer unlock];
  38.     outputFramebuffer = nil;
  40.     // Finally, trigger rendering as needed
  41.     for (id<GPUImageInput> currentTarget in targets)
  42.     {
  43.         if ([currentTarget enabled])
  44.         {
  45.             NSInteger indexOfObject = [targets indexOfObject:currentTarget];
  46.             NSInteger textureIndexOfTarget = [[targetTextureIndices objectAtIndex:indexOfObject] integerValue];
  48.             if (currentTarget != self.targetToIgnoreForUpdates)
  49.             {
  50.                 [currentTarget newFrameReadyAtTime:currentTime atIndex:textureIndexOfTarget];
  51.             }
  52.         }
  53.     }
  54. }

GPUImageFilter

上面讲了输入链条里的输入源，这里讲的就是输入链条中间的处理部分。

GPUImageFilter就是用来接收源图像，通过自定义的顶点、片元着色器来渲染新的图像，并在绘制完成后通知响应链的下一个对象。

GPUImageFilter和响应链的其他元素实现了GPUImageInput协议，他们都可以提供纹理参与响应链，或者从响应链的前面接收并处理纹理。响应链的下一个对象是target，响应链可能有多个分支（添加多个targets）。

响应链上一级通知渲染

上面说过输入源输入缓冲帧准备好之后通过调用- (void)newFrameReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex方法来通知target准备渲染。

  1. - (void)newFrameReadyAtTime:(CMTime)frameTime atIndex:(NSInteger)textureIndex
  2. {
  3.     // 顶点坐标系矩阵
  4.     static const GLfloat imageVertices[] = {
  5.         -1.0f, -1.0f,
  6.         1.0f, -1.0f,
  7.         -1.0f,  1.0f,
  8.         1.0f,  1.0f,
  9.     };
  11.     // 绘制结果输出
  12.     [self renderToTextureWithVertices:imageVertices textureCoordinates:[[self class] textureCoordinatesForRotation:inputRotation]];
  14.     // 通知targets
  15.     [self informTargetsAboutNewFrameAtTime:frameTime];
  16. }

获取顶点坐标和纹理坐标

以下为固定的顶点坐标：

  1. static const GLfloat imageVertices[] = {
  2.     -1.0f, -1.0f,
  3.     1.0f, -1.0f,
  4.     -1.0f,  1.0f,
  5.     1.0f,  1.0f,
  6. };

通过这个方法获取纹理坐标，根据传进来的选择类型选择纹理坐标:

  1. + (const GLfloat *)textureCoordinatesForRotation:(GPUImageRotationMode)rotationMode{
  2.     // 正常不旋转
  3.     static const GLfloat noRotationTextureCoordinates[] = {
  4.         0.0f, 0.0f,
  5.         1.0f, 0.0f,
  6.         0.0f, 1.0f,
  7.         1.0f, 1.0f,
  8.     };
  10.     // 图片向右旋转
  11.     static const GLfloat rotateLeftTextureCoordinates[] = {
  12.         1.0f, 0.0f,
  13.         1.0f, 1.0f,
  14.         0.0f, 0.0f,
  15.         0.0f, 1.0f,
  16.     };
  18.     .........
  19. }

绘制结果输出

  1. // 渲染到帧缓存
  2. - (void)renderToTextureWithVertices:(const GLfloat *)vertices textureCoordinates:(const GLfloat *)textureCoordinates;
  3. {
  4.     // 是否阻止渲染
  5.     if (self.preventRendering)
  6.     {
  7.         // 解锁帧缓存
  8.         [firstInputFramebuffer unlock];
  9.         return;
  10.     }
  12.     [GPUImageContext setActiveShaderProgram:filterProgram];
  14.     // 自己的输出帧缓存
  15.     outputFramebuffer = [[GPUImageContext sharedFramebufferCache] fetchFramebufferForSize:[self sizeOfFBO] textureOptions:self.outputTextureOptions onlyTexture:NO];
  16.     [outputFramebuffer activateFramebuffer];
  17.     if (usingNextFrameForImageCapture)
  18.     {
  19.         [outputFramebuffer lock];
  20.     }
  22.     [self setUniformsForProgramAtIndex:0];
  24.     glClearColor(backgroundColorRed, backgroundColorGreen, backgroundColorBlue, backgroundColorAlpha);
  25.     glClear(GL_COLOR_BUFFER_BIT);
  27.     // 绑定输入纹理
  28.     glActiveTexture(GL_TEXTURE2);
  29.     glBindTexture(GL_TEXTURE_2D, [firstInputFramebuffer texture]);
  30.     // 把位置2的纹理单元赋值给采样器（告诉GLSL选择的纹理单元是2）
  31.     glUniform1i(filterInputTextureUniform, 2);  
  33.     // 绑定顶点和纹理坐标并绘制图元
  34.     glVertexAttribPointer(filterPositionAttribute, 2, GL_FLOAT, 0, 0, vertices);
  35.     glVertexAttribPointer(filterTextureCoordinateAttribute, 2, GL_FLOAT, 0, 0, textureCoordinates);
  36.     glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
  38.     // 解锁输入帧缓存对象
  39.     [firstInputFramebuffer unlock];
  41.     // 需要等待绘制完成才去生成图像（默认为NO）
  42.     if (usingNextFrameForImageCapture)
  43.     {
  44.         // 发送绘制完成信号
  45.         dispatch_semaphore_signal(imageCaptureSemaphore);
  46.     }
  47. }

通知下一级targets绘制渲染

这一步基本就跟上面输入源更新所有的targets的输入参数，并通知渲染差不多了，一样的步骤：

  1. - (void)informTargetsAboutNewFrameAtTime:(CMTime)frameTime;

参考

GPUImage
GPUImage源码阅读

添加新批注
在作者公开此批注前，只有你和作者可见。
回复批注