intel硬件视频加速介绍

硬件视频加速技术


硬件视频加速(Hardware video acceleration) 通过让显卡编解码视频,从而减轻cpu负担并且还能节省电源。

在linux上的几种实现:
Video Acceleration API (VAAPI)英特尔开发的规范和开源库,提供硬件加速视频的编/解码。
Video Decode and Presentation API for Unix (VDPAU)是一套NVIDIA开发的,用于将部分视频解码和渲染分担给GPU来处理的开源库和API。

NVDECODE/NVENCODENVIDIA Fermi, Kepler, Maxwell and Pascal系列显卡专有的硬件加速API

intel 硬件加速技术

intel 的开源媒体栈

MSDK : ​https://github.com/Intel-Media-SDK/MediaSDK
Libva : https://github.com/intel/libva
i965 driver : ​https://github.com/intel/intel-vaapi-driver
iHD driver : ​https://github.com/intel/media-driver

VA-API

开源驱动:

AMD: 位于官方软件仓库的libva-vdpau-drivermesa两者为 Radeon 9500 或更新的GPU提供支持。
Intel: 位于官方软件仓库的libva-intel-driver软件包为 GMA 4500 系列或者更新的GPU提供支持。
NVIDIA: 位于官方软件仓库的libva-vdpau-drivermesa两者为 GeForce 8 系列和更新的GPU提供支持。位于AUR的 nouveau-fwAUR 软件包包含从NVIDIA闭源驱动中提取出的必要的固件文件。

闭源驱动:

AMD: 位于AUR的libva-xvba-driverAUR[broken link: archived in aur-mirror]软件包为 Radeon HD 4000 系列或更新的GPU提供支持。在 Radeon HD 5000 系列或者更新的GPU上请使用 catalyst-utilsAUR 驱动;在 Radeon HD 4000 系列上使用 catalyst-total-hd234kAUR 作为驱动程序。
NVIDIA: 位于官方软件仓库的libva-vdpau-driver软件包和nvidia-utils驱动为 GeForce 8 系列或更新的GPU提供支持。

intel 依赖libva-intel-driver

AMD 依赖libva-mesa-driver)

安装

以前整理过ubuntu下的简洁安装记录:
linux 下 intel vaapi安装

主要是安装media-driver及其依赖库。

检验 VA-API
运行 libva-utils 包提供的 vainfo工具 来检查 VA-API 的设置:

$ vainfo  libva info: VA-API version 0.39.4  libva info: va_getDriverName() returns 0  libva info: Trying to open /usr/lib/dri/i965_drv_video.so  libva info: Found init function __vaDriverInit_0_39  libva info: va_openDriver() returns 0  vainfo: VA-API version: 0.39 (libva 1.7.3)  vainfo: Driver version: Intel i965 driver for Intel(R) Skylake - 1.7.3  vainfo: Supported profile and entrypoints        VAProfileMPEG2Simple            : VAEntrypointVLD        VAProfileMPEG2Simple            : VAEntrypointEncSlice        VAProfileMPEG2Main              : VAEntrypointVLD        VAProfileMPEG2Main              : VAEntrypointEncSlice        VAProfileH264ConstrainedBaseline: VAEntrypointVLD        VAProfileH264ConstrainedBaseline: VAEntrypointEncSlice        VAProfileH264ConstrainedBaseline: VAEntrypointEncSliceLP        VAProfileH264Main               : VAEntrypointVLD        VAProfileH264Main               : VAEntrypointEncSlice        VAProfileH264Main               : VAEntrypointEncSliceLP        VAProfileH264High               : VAEntrypointVLD        VAProfileH264High               : VAEntrypointEncSlice        VAProfileH264High               : VAEntrypointEncSliceLP        VAProfileH264MultiviewHigh      : VAEntrypointVLD        VAProfileH264MultiviewHigh      : VAEntrypointEncSlice        VAProfileH264StereoHigh         : VAEntrypointVLD        VAProfileH264StereoHigh         : VAEntrypointEncSlice        VAProfileVC1Simple              : VAEntrypointVLD        VAProfileVC1Main                : VAEntrypointVLD        VAProfileVC1Advanced            : VAEntrypointVLD        VAProfileNone                   : VAEntrypointVideoProc        VAProfileJPEGBaseline           : VAEntrypointVLD        VAProfileJPEGBaseline           : VAEntrypointEncPicture        VAProfileVP8Version0_3          : VAEntrypointVLD        VAProfileVP8Version0_3          : VAEntrypointEncSlice        VAProfileHEVCMain               : VAEntrypointVLD        VAProfileHEVCMain               : VAEntrypointEncSlice

样例

ffmpeg vaapi h264加速样例,其他ffmpeg wiki还有很多。

$ ffmpeg -threads 1 -i file.ext -vaapi_device  /dev/dri/renderD128 -vcodec h264_vaapi -vf format='nv12|vaapi,hwupload' output.mp4

代码例
曾经写过的vaapi样例: 解码->overlay->编码

package main    import(      "github.com/ailumiyana/goav-incr/goav/avcodec"      "github.com/ailumiyana/goav-incr/goav/avutil"      "github.com/ailumiyana/goav-incr/goav/avfilter"        "github.com/ailumiyana/tools/latency"      log "github.com/astaxie/beego/logs"        "fmt"      //"time"      "io/ioutil"      "os"      "strconv"      "unsafe"  )      var input_h264  string = "1080p.h264"  var output_h264 string = "out.h264"    type VaapiHWDeviceCtx struct {      hw_device_ctx *avutil.BufferRef  }    // input : "/dev/dri/card0" or "/dev/dri/renderD128"  func Create(device string) *VaapiHWDeviceCtx {      var hw_device_ctx *avutil.BufferRef      err := avutil.AVHwdeviceCtxCreate(&hw_device_ctx, avutil.AV_HWDEVICE_TYPE_VAAPI,          device, nil, 0)      if err < 0 {          fmt.Println("AVHwdeviceCtxCreate err : ", avutil.ErrorFromCode(err))          return nil      }        return &VaapiHWDeviceCtx{          hw_device_ctx,      }  }    func (v *VaapiHWDeviceCtx)Context() *avutil.BufferRef{      return v.hw_device_ctx  }    //warning need unref by avutil.AVBufferUnref()  func (v *VaapiHWDeviceCtx)GetAnAllocHwframeCtxRef(format, sw_format avutil.PixelFormat, w, h int) *avutil.BufferRef{      hw_frames_ref := avutil.AVHwframeCtxAlloc(v.hw_device_ctx)      if hw_frames_ref == nil {          fmt.Println("AVHwframeCtxAlloc err")          return nil      }        frames_ctx := hw_frames_ref.HWFramesContext()      //frames_ctx.SetHWFramesContextPrarms(avutil.PixelFormat(avcodec.AV_PIX_FMT_VAAPI),      //      avutil.PixelFormat(avcodec.AV_PIX_FMT_YUV420P),      //      1280, 720)      frames_ctx.SetHWFramesContextPrarms(format, sw_format, w, h)        err := avutil.AVHwFrameCtxInit(hw_frames_ref)      if err < 0 {          fmt.Println("AVHwFrameCtxInit err : ", avutil.ErrorFromCode(err))          avutil.AVBufferUnref(&hw_frames_ref)          return nil      }        return hw_frames_ref  }    //var hw_device_ctx *avutil.BufferRef// = avutil.AVHwdeviceCtxAlloc(avutil.AV_HWDEVICE_TYPE_VAAPI)    func SetHwframesContext(ctxt *avcodec.Context) {      /*var hw_device_ctx *avutil.BufferRef        err := avutil.AVHwdeviceCtxCreate(&hw_device_ctx, avutil.AV_HWDEVICE_TYPE_VAAPI,          "/dev/dri/card0", nil, 0)      if err < 0 {          fmt.Println("AVHwdeviceCtxCreate err : ", avutil.ErrorFromCode(err))          panic("AVHwdeviceCtxCreate faild")      }*/        vaapi_device  := Create("/dev/dri/card0")      //hw_device_ctx := vaapi_device.Context()        /*hw_frames_ref := avutil.AVHwframeCtxAlloc(hw_device_ctx)      if hw_frames_ref == nil {          fmt.Println("AVHwframeCtxAlloc err")          panic("AVHwframeCtxAlloc faild")      }        frames_ctx := hw_frames_ref.HWFramesContext()      frames_ctx.SetHWFramesContextPrarms(avutil.PixelFormat(avcodec.AV_PIX_FMT_VAAPI),              avutil.PixelFormat(avcodec.AV_PIX_FMT_YUV420P),              1280, 720)        err := avutil.AVHwFrameCtxInit(hw_frames_ref)      if err < 0 {          fmt.Println("AVHwFrameCtxInit err : ", avutil.ErrorFromCode(err))          avutil.AVBufferUnref(&hw_frames_ref)          panic("AVHwFrameCtxInit faild")      }*/        hw_frames_ref := vaapi_device.GetAnAllocHwframeCtxRef(                              avutil.PixelFormat(avcodec.AV_PIX_FMT_VAAPI),                              avutil.PixelFormat(avcodec.AV_PIX_FMT_YUV420P), 1280, 720)        //enc.Context().HWFramesCtx() = avutil.AVBufferRef(hw_frames_ref)      ctxt.SetHWFramesCtx(avutil.AVBufferRef(hw_frames_ref))      if ctxt.HWFramesCtx() == nil {          fmt.Println("SetHWFrameCtx err ")          avutil.AVBufferUnref(&hw_frames_ref)          panic("SetHWFrameCtx faild")      }        //fmt.Println(frames_ctx)      //fmt.Println(hw_frames_ref)      //fmt.Println(ctxt.HWFramesCtx())        avutil.AVBufferUnref(&hw_frames_ref)  }      func decode_encode() {      //decoder      vaapi_device  := Create("/dev/dri/renderD128")        pkt            := avcodec.AvPacketAlloc()      if pkt == nil {          log.Critical("AvPacketAlloc failed.")          return      }        codec          := avcodec.AvcodecFindDecoder(avcodec.CodecId(avcodec.AV_CODEC_ID_H264))      if codec == nil {          log.Critical("AvcodecFindDecoder failed.")      }        context        := codec.AvcodecAllocContext3()      if context == nil {          log.Critical("AvcodecAllocContext3 failed.")          return      }        parserContext  := avcodec.AvParserInit(int(avcodec.CodecId(avcodec.AV_CODEC_ID_H264)))      if parserContext == nil {          log.Critical("AvParserInit failed.")          return      }        frame          := avutil.AvFrameAlloc()      if frame == nil {          log.Critical("AvFrameAlloc failed.")          return      }        context.SetHWDeviceCtx(avutil.AVBufferRef(vaapi_device.Context()))      err := context.AvcodecOpen2(codec, nil)      if err < 0 {          log.Critical("AvcodecOpen2 failed.")          return      }        // encoder      codec_enc          := avcodec.AvcodecFindEncoderByName("h264_vaapi")      if codec_enc == nil {          log.Critical("AvcodecFindEncoderByName failed.")      }        pkt_enc            := avcodec.AvPacketAlloc()      if pkt_enc == nil {          log.Critical("AvPacketAlloc failed.")      }        context_enc        := codec_enc.AvcodecAllocContext3()      if context_enc == nil {          log.Critical("AvcodecAllocContext3 failed.")      }        context_enc.SetVideoEncodeParams(2000000, 1920, 1080,          avcodec.AV_PIX_FMT_VAAPI,          false, 10)      context_enc.SetTimebase(1, 30)        hw_frames_ref := vaapi_device.GetAnAllocHwframeCtxRef(                              avutil.PixelFormat(avcodec.AV_PIX_FMT_VAAPI),                              avutil.PixelFormat(avcodec.AV_PIX_FMT_NV12), 1920, 1080)          context_enc.SetHWFramesCtx(avutil.AVBufferRef(hw_frames_ref))        err = context_enc.AvcodecOpen2(codec_enc, nil)      if err < 0 {          log.Critical("AvcodecOpen2 failed.")          return      }        decLatency := latency.New("vaapi", "decode")      encLatency := latency.New("vaapi", "encode")      /*scaleLatency := latency.New("vaapi", "scale")      overlayLatency := latency.New("vaapi", "overlay")*/        data, erro := ioutil.ReadFile(input_h264)      if erro != nil {          log.Debug("File reading error", erro)          return      }      log.Debug("Open Success.")      l := len(data)      log.Debug("size of file:", l)        b := make([]byte, 4096 + 64)        file, erro := os.Create(output_h264)      if erro != nil {          log.Critical("Error Reading")      }      defer file.Close()        //var pts int64 = 0      //var opts int64 = 0        //var frames [25]*avutil.Frame      started := false      sum := 0      for sum < l {          remain := 4096          for remain > 0 {              copy(b, data[sum:sum + 4096])              if !started {                  decLatency.Start()                  started = true              }              n := context.AvParserParse2(parserContext, pkt, b,                  remain, avcodec.AV_NOPTS_VALUE, avcodec.AV_NOPTS_VALUE, 0)              log.Debug("parser ", n, "bytes")                sum     = sum + n              remain  = remain - n;                //log.Trace("--------", dec.Packet().GetPacketSize())              if pkt.GetPacketSize() > 0 {                  //fmt.Println(decLatency.End())                    ret := context.AvcodecSendPacket(pkt)                  if ret < 0 {                      log.Error("AvcodecSendPacket err ", avutil.ErrorFromCode(ret))                      continue                  }                    ret = context.AvcodecReceiveFrame((*avcodec.Frame)(unsafe.Pointer(frame)))                  if ret < 0 {                      log.Error("AvcodecReceiveFrame err ", avutil.ErrorFromCode(ret))                      continue                  }                    if ret == 0 {                      started = false                      decLatency.End()                      fmt.Println(decLatency.End())                        log.Debug("dec-frame:",frame)                          encLatency.Start()                      ret = context_enc.AvcodecSendFrame((*avcodec.Frame)(unsafe.Pointer(frame)))                      if ret < 0 {                          log.Trace("AvcodecSendFrame err ", avutil.ErrorFromCode(ret))                          continue                      }                        ret = context_enc.AvcodecReceivePacket(pkt_enc)                      if ret < 0 {                          log.Trace("AvcodecReceivePacket err ", avutil.ErrorFromCode(ret))                          continue                      }                      if ret == 0 {                          data0 := pkt_enc.Data()                          buf := make([]byte, pkt_enc.GetPacketSize())                          start := uintptr(unsafe.Pointer(data0))                          for i := 0; i < pkt_enc.GetPacketSize(); i++ {                              elem := *(*uint8)(unsafe.Pointer(start + uintptr(i)))                              buf[i] = elem                          }                            file.Write(buf)                          //encLatency.End()                          fmt.Println(encLatency.End())                      }                        avutil.AvFrameUnref(frame)                      //avutil.AvFrameUnref(oframe)                    }                }          }      }  }    func decode_overlay_encode() {      //decoder      vaapi_device  := Create("/dev/dri/renderD128")        pkt            := avcodec.AvPacketAlloc()      if pkt == nil {          log.Critical("AvPacketAlloc failed.")          return      }        codec          := avcodec.AvcodecFindDecoder(avcodec.CodecId(avcodec.AV_CODEC_ID_H264))      if codec == nil {          log.Critical("AvcodecFindDecoder failed.")      }        context        := codec.AvcodecAllocContext3()      if context == nil {          log.Critical("AvcodecAllocContext3 failed.")          return      }        parserContext  := avcodec.AvParserInit(int(avcodec.CodecId(avcodec.AV_CODEC_ID_H264)))      if parserContext == nil {          log.Critical("AvParserInit failed.")          return      }        frame          := avutil.AvFrameAlloc()      if frame == nil {          log.Critical("AvFrameAlloc failed.")          return      }        context.SetHWDeviceCtx(avutil.AVBufferRef(vaapi_device.Context()))      err := context.AvcodecOpen2(codec, nil)      if err < 0 {          log.Critical("AvcodecOpen2 failed.")          return      }        // encoder      codec_enc          := avcodec.AvcodecFindEncoderByName("h264_vaapi")      if codec_enc == nil {          log.Critical("AvcodecFindEncoderByName failed.")      }        pkt_enc            := avcodec.AvPacketAlloc()      if pkt_enc == nil {          log.Critical("AvPacketAlloc failed.")      }        context_enc        := codec_enc.AvcodecAllocContext3()      if context_enc == nil {          log.Critical("AvcodecAllocContext3 failed.")      }        context_enc.SetVideoEncodeParams(2000000, 1920, 1080,          avcodec.AV_PIX_FMT_VAAPI,          false, 10)      context_enc.SetTimebase(1, 30)        hw_frames_ref := vaapi_device.GetAnAllocHwframeCtxRef(                              avutil.PixelFormat(avcodec.AV_PIX_FMT_VAAPI),                              avutil.PixelFormat(avcodec.AV_PIX_FMT_NV12), 1920, 1080)          /*context_enc.SetHWFramesCtx(avutil.AVBufferRef(outs[0].AvBuffersinkGetHwFramesCtx()))        err = context_enc.AvcodecOpen2(codec_enc, nil)      if err < 0 {          log.Critical("AvcodecOpen2 failed.")          return      }*/        decLatency := latency.New("vaapi", "decode")      encLatency := latency.New("vaapi", "encode")        // filter      args := "video_size=1920x1080:pix_fmt=46:time_base=1/30:pixel_aspect=1/1"      des  := "[in0] fps=30,scale_vaapi=w=960:h=540:format=nv12, hwdownload,format=nv12 [s0];color=black:r=30:size=1920x1080:sar=1/1 [b0];[b0][s0] overlay=x=0:y=0, format=nv12, hwupload"        graph := avfilter.AvfilterGraphAlloc()      if graph == nil {          log.Critical("AvfilterGraphAlloc Failed.")          return      }      //graph.SetNbThreads(8)        inputs  := avfilter.AvfilterInoutAlloc()      outputs := avfilter.AvfilterInoutAlloc()      if inputs == nil || outputs == nil {          log.Critical("AvfilterInoutAlloc Failed.")          return      }        defer avfilter.AvfilterInoutFree(inputs)      defer avfilter.AvfilterInoutFree(outputs)        var buffersrc *avfilter.Filter      var buffersink *avfilter.Filter      if false {          buffersrc  = avfilter.AvfilterGetByName("abuffer")          buffersink = avfilter.AvfilterGetByName("abuffersink")      } else {          buffersrc  = avfilter.AvfilterGetByName("buffer")          buffersink = avfilter.AvfilterGetByName("buffersink")      }        if buffersink == nil || buffersrc == nil {          log.Critical("AvfilterGetByName Failed.")          return      }        ret := graph.AvfilterGraphParse2(des, &inputs, &outputs)      if ret < 0 {          log.Critical("AvfilterInoutAlloc Failed des : ", avutil.ErrorFromCode(ret))          return      }        if vaapi_device.Context() != nil {          for _, v := range graph.Filters() {              v.SetHWDeviceCtx(avutil.AVBufferRef(vaapi_device.Context()))          }      }        var ins    []*avfilter.Context      var outs   []*avfilter.Context      var frames []*avutil.Frame        // inputs      index := 0        for cur := inputs; cur != nil; cur = cur.Next() {          //log.Debug("index :", index)          var in *avfilter.Context          //var args = "video_size=1280x720:pix_fmt=0:time_base=1/30:pixel_aspect=1/1"          inName := "in" + strconv.Itoa(index)          ret = avfilter.AvfilterGraphCreateFilter(&in, buffersrc, inName, args, 0, graph)          if ret < 0 {              log.Critical("AvfilterGraphCreateFilter Failed des : ", avutil.ErrorFromCode(ret))              return          }            par := avfilter.AvBuffersrcParametersAlloc()          if par == nil {              log.Critical("AvBuffersrcParametersAlloc Failed.")              return          }          par.SetHwFramesCtx(avutil.AVBufferRef(hw_frames_ref))          ret = in.AvBuffersrcParametersSet(par)          if ret < 0 {              log.Critical("AvBuffersrcParametersSet Failed.")              return          }          avutil.AvFreep(unsafe.Pointer(&par))            ins = append(ins, in)          ret = avfilter.AvfilterLink(ins[index], 0, cur.FilterContext(), cur.PadIdx())          if ret < 0 {              log.Critical("AvfilterLink Failed des : ", avutil.ErrorFromCode(ret))              return          }          index++      }        // outputs      index = 0      for cur := outputs; cur != nil; cur = cur.Next() {          var out *avfilter.Context          outName := "out" + strconv.Itoa(index)          ret = avfilter.AvfilterGraphCreateFilter(&out, buffersink, outName, "", 0, graph)          if ret < 0 {              log.Critical("AvfilterGraphCreateFilter Failed des : ", avutil.ErrorFromCode(ret))              return          }            outs = append(outs, out)          ret = avfilter.AvfilterLink(cur.FilterContext(), cur.PadIdx(), outs[index], 0)          if ret < 0 {              log.Critical("AvfilterLink Failed des : ", avutil.ErrorFromCode(ret))              return          }          index++            f := avutil.AvFrameAlloc()          if f == nil {              log.Critical("AvFrameAlloc failed.")              return          }          frames = append(frames, f)      }        ret = graph.AvfilterGraphConfig(0)      if ret < 0 {          log.Critical("AvfilterGraphConfig Failed des : ", avutil.ErrorFromCode(ret))          return      }        overlayLatency := latency.New("vaapi", "overlay")          context_enc.SetHWFramesCtx(avutil.AVBufferRef(outs[0].AvBuffersinkGetHwFramesCtx()))        err = context_enc.AvcodecOpen2(codec_enc, nil)      if err < 0 {          log.Critical("AvcodecOpen2 failed.")          return      }        data, erro := ioutil.ReadFile(input_h264)      if erro != nil {          log.Debug("File reading error", erro)          return      }      log.Debug("Open Success.")      l := len(data)      log.Debug("size of file:", l)        b := make([]byte, 4096 + 64)        file, erro := os.Create(output_h264)      if erro != nil {          log.Critical("Error Reading")      }      defer file.Close()        var pts int64 = 0      //var opts int64 = 0        //var frames [25]*avutil.Frame      started := false      sum := 0      for sum < l {          remain := 4096          for remain > 0 {              copy(b, data[sum:sum + 4096])              if !started {                  decLatency.Start()                  started = true              }              n := context.AvParserParse2(parserContext, pkt, b,                  remain, avcodec.AV_NOPTS_VALUE, avcodec.AV_NOPTS_VALUE, 0)              log.Debug("parser ", n, "bytes")                sum     = sum + n              remain  = remain - n;                //log.Trace("--------", dec.Packet().GetPacketSize())              if pkt.GetPacketSize() > 0 { // decode                  //fmt.Println(decLatency.End())                    ret := context.AvcodecSendPacket(pkt)                  if ret < 0 {                      log.Error("AvcodecSendPacket err ", avutil.ErrorFromCode(ret))                      continue                  }                    ret = context.AvcodecReceiveFrame((*avcodec.Frame)(unsafe.Pointer(frame)))                  if ret < 0 {                      log.Error("AvcodecReceiveFrame err ", avutil.ErrorFromCode(ret))                      continue                  }                    if ret == 0 {                      decLatency.End()                      fmt.Println(decLatency.End())                  } else {                      continue                  }                    overlayLatency.Start()                  pts++                  frame.SetPts(pts)                    ret = avfilter.AvBuffersrcAddFrame(ins[0], (*avfilter.Frame)(unsafe.Pointer(frame)))                  if ret < 0 {                      fmt.Println("AvBuffersrcAddFrame error,", avutil.ErrorFromCode(ret))                      continue                  }                    ret = avfilter.AvBufferSinkGetFrame(outs[0], (*avfilter.Frame)(unsafe.Pointer(frames[0])))                    if ret == avutil.AvErrorEOF || ret == avutil.AvErrorEAGAIN {                      log.Error(avutil.ErrorFromCode(ret))                      continue                  }                    if ret < 0 {                      log.Error("AvBufferSinkGetFrame Failed des : ", ret, avutil.ErrorFromCode(ret))                      continue                  }                    frame = frames[0]                    fmt.Println(overlayLatency.End())                    if ret == 0 { // encode                      started = false                        //log.Debug("dec-frame:",frame)                        encLatency.Start()                      ret = context_enc.AvcodecSendFrame((*avcodec.Frame)(unsafe.Pointer(frame)))                      if ret < 0 {                          log.Trace("AvcodecSendFrame err ", avutil.ErrorFromCode(ret))                          continue                      }                        ret = context_enc.AvcodecReceivePacket(pkt_enc)                      if ret < 0 {                          log.Trace("AvcodecReceivePacket err ", avutil.ErrorFromCode(ret))                          continue                      }                      if ret == 0 {                          data0 := pkt_enc.Data()                          buf := make([]byte, pkt_enc.GetPacketSize())                          start := uintptr(unsafe.Pointer(data0))                          for i := 0; i < pkt_enc.GetPacketSize(); i++ {                              elem := *(*uint8)(unsafe.Pointer(start + uintptr(i)))                              buf[i] = elem                          }                            file.Write(buf)                          //encLatency.End()                          fmt.Println(encLatency.End())                      }                        avutil.AvFrameUnref(frame)                      //avutil.AvFrameUnref(oframe)                    }                }          }      }  }    func main() {      //decode_encode()      decode_overlay_encode()  }

Intel Quick Sync(QSV)

除了VAAPI之外,intel还有一项自己平台专属的硬件加速技术QSV(quick-sync-video)

Quick Sync Video 使用Intel图形技术专用媒体处理能力来快速编解码。
ffmpeg wiki有一页qsv的介绍,但是推荐使用VA-API(iHD / i965) 驱动代替libmfx.

API支持情况

QSV支持访问多种不同的媒体库API:
DXVA2 / D3D11VA : 标准Windows api,由Intel图形驱动程序实现,以支持视频解码。

linux libmfx:这是intel的一个库,可以作为Intel Media SDK的一部分安装,并且支持一些编码和解码类型。

Windows libmfx: 英特尔的图形驱动程序提供,它支持所有的编码和解码的类型。

Media Foundation:通过英特尔图形驱动程序,支持一部分编解码类型,没有继承进ffmpeg。

VAAPI i965驱动:这是一个几乎免费的libva/VAAPI驱动程序。大多数Linux发行版都打包了它。

VAAPI iHD驱动:Linux上的libmfx后期使用了的一个修改过的libva和VAAPI驱动程序;这也可以由用户直接使用。

vaapi/mfx比较

vaapi/i965

1)在大多数Linux发行版中都是作为标准打包的。
2)可运行在所有可用的硬件上,包括一些较老和较便宜的设备。
3)支持大多数编解码器。
4)通用的API,也可以使用基于Mesa的AMD / Nvidia硬件。
5)可与标准api (EGL/OpenGL, OpenCL)交互。

libmfx /iHD

1)在某些情况下,可能会提供更好的编码质量(如look_ahead)。
2)在某些情况下(如MFE,特别是在Iris图形上)可能会提供更高的编码吞吐量。
3)linux和windows皆通用的API。
4)可与Intel实现的OpenCL实现交互。

安装

linux
可以安装基于vaapi开源的MediaSDK

windows
官方有驱动下载:https://software.intel.com/media-sdk

样例

h264解码

ffmpeg -hwaccel qsv -c:v h264_qsv -i input.mp4 -f null -

代码例
qsv 解码->overlay->编码

package main    import (      log "github.com/astaxie/beego/logs"      "github.com/ailumiyana/goav-incr/goav/avutil"      "github.com/ailumiyana/goav-incr/goav/avcodec"      "github.com/ailumiyana/goav-incr/goav/avfilter"        "github.com/ailumiyana/tools/latency"        "fmt"      "time"      "io/ioutil"      "os"      "strconv"      "unsafe"  )    var input_h264  string = "1080p.h264"  var input_h264_720  string = "recv0.h264"  var output_h264 string = "out.h264"      var overlay16 string = "color=white:r=30:size=1920x1080:sar=1/1,hwupload=extra_hw_frames=300,format=qsv [background];" +  "[in0] scale_qsv=479:269 [in_0];" +  "[in1] scale_qsv=479:269 [in_1];" +  "[in2] scale_qsv=479:269 [in_2];" +  "[in3] scale_qsv=479:269 [in_3];" +  "[in4] scale_qsv=479:269 [in_4];" +  "[in5] scale_qsv=479:269 [in_5];" +  "[in6] scale_qsv=479:269 [in_6];" +  "[in7] scale_qsv=479:269 [in_7];" +  "[in8] scale_qsv=479:269 [in_8];" +  "[in9] scale_qsv=479:269 [in_9];" +  "[in10] scale_qsv=479:269 [in_10];" +  "[in11] scale_qsv=479:269 [in_11];" +  "[in12] scale_qsv=479:269 [in_12];" +  "[in13] scale_qsv=479:269 [in_13];" +  "[in14] scale_qsv=479:269 [in_14];" +  "[in15] scale_qsv=479:269 [in_15];" +  "[background][in_0] overlay_qsv=x=0:y=0 [background+in0_scale];" +  "[background+in0_scale][in_1] overlay_qsv=x=481:y=0 [background+in0_scale+in1_scale];" +  "[background+in0_scale+in1_scale][in_2] overlay_qsv=x=961:y=0 [background+in0_scale+in1_scale+in2_scale];" +  "[background+in0_scale+in1_scale+in2_scale][in_3] overlay_qsv=x=1441:y=0 [background+in0_scale+in1_scale+in2_scale+in3_scale];" +  "[background+in0_scale+in1_scale+in2_scale+in3_scale][in_4] overlay_qsv=x=0:y=271 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale];" +  "[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale][in_5] overlay_qsv=x=481:y=271 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale];" +  "[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale][in_6] overlay_qsv=x=961:y=271 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale];" +  "[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale][in_7] overlay_qsv=x=1441:y=271 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale];" +  "[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale][in_8] overlay_qsv=x=0:y=541 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale];" +  "[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale][in_9] overlay_qsv=x=481:y=541 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale];" +  "[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale][in_10] overlay_qsv=x=961:y=541 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale];" +  "[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale][in_11] overlay_qsv=x=1441:y=541 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale];" +  "[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale][in_12] overlay_qsv=x=0:y=811 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale+in12_scale];" +  "[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale+in12_scale][in_13] overlay_qsv=x=481:y=811 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale+in12_scale+in13_scale];" +  "[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale+in12_scale+in13_scale][in_14] overlay_qsv=x=961:y=811 [background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale+in12_scale+in13_scale+in14_scale];" +  "[background+in0_scale+in1_scale+in2_scale+in3_scale+in4_scale+in5_scale+in6_scale+in7_scale+in8_scale+in9_scale+in10_scale+in11_scale+in12_scale+in13_scale+in14_scale][in_15] overlay_qsv=x=1441:y=811"      type QsvHWDeviceCtx struct {      hw_device_ctx *avutil.BufferRef  }    // input : "/dev/dri/card0" or "/dev/dri/renderD128"  func Create(device string) *QsvHWDeviceCtx {      var hw_device_ctx *avutil.BufferRef      err := avutil.AVHwdeviceCtxCreate(&hw_device_ctx, avutil.AV_HWDEVICE_TYPE_QSV,          device, nil, 0)      if err < 0 {          log.Error("AVHwdeviceCtxCreate err : ", avutil.ErrorFromCode(err))          return nil      }        return &QsvHWDeviceCtx{          hw_device_ctx,      }  }    func (v *QsvHWDeviceCtx) Context() *avutil.BufferRef{      return v.hw_device_ctx  }    //warning need unref by avutil.AVBufferUnref()  func (v *QsvHWDeviceCtx)GetAnAllocHwframeCtxRef(format, sw_format avutil.PixelFormat, w, h, s int) *avutil.BufferRef{      hw_frames_ref := avutil.AVHwframeCtxAlloc(v.hw_device_ctx)      if hw_frames_ref == nil {          log.Error("AVHwframeCtxAlloc err")          return nil      }        frames_ctx := hw_frames_ref.HWFramesContext()        frames_ctx.SetQsvHWFramesContextPrarms(format, sw_format, w, h, s)        err := avutil.AVHwFrameCtxInit(hw_frames_ref)      if err < 0 {          log.Error("AVHwFrameCtxInit err : ", avutil.ErrorFromCode(err))          avutil.AVBufferUnref(&hw_frames_ref)          return nil      }        return hw_frames_ref  }        func decode_overlay_encode() {      //decoder      qsv_device  := Create("/dev/dri/renderD128")        pkt            := avcodec.AvPacketAlloc()      if pkt == nil {          log.Critical("AvPacketAlloc failed.")          return      }        codec        := avcodec.AvcodecFindDecoderByName("h264_qsv")      if codec == nil {          log.Critical("AvcodecFindDecoderByName failed.")          return      }        context        := codec.AvcodecAllocContext3()      if context == nil {          log.Critical("AvcodecAllocContext3 failed.")          return      }        parserContext  := avcodec.AvParserInit(int(avcodec.CodecId(avcodec.AV_CODEC_ID_H264)))      if parserContext == nil {          log.Critical("AvParserInit failed.")          return      }        frame          := avutil.AvFrameAlloc()      if frame == nil {          log.Critical("AvFrameAlloc failed.")          return      }        context.SetHWDeviceCtx(avutil.AVBufferRef(qsv_device.Context()))      err := context.AvcodecOpen2(codec, nil)      if err < 0 {          log.Critical("AvcodecOpen2 failed.")          return      }      context.SetDefaultQsvGetFormat()        // encoder      codec_enc          := avcodec.AvcodecFindEncoderByName("h264_qsv")      if codec_enc == nil {          log.Critical("AvcodecFindEncoderByName failed.")      }        pkt_enc            := avcodec.AvPacketAlloc()      if pkt_enc == nil {          log.Critical("AvPacketAlloc failed.")      }        context_enc        := codec_enc.AvcodecAllocContext3()      if context_enc == nil {          log.Critical("AvcodecAllocContext3 failed.")      }        context_enc.SetVideoEncodeParams(2000000, 1920, 1080,          avcodec.AV_PIX_FMT_QSV,          false, 10)      context_enc.SetTimebase(1, 30)        hw_frames_ref := qsv_device.GetAnAllocHwframeCtxRef(                              avutil.PixelFormat(avcodec.AV_PIX_FMT_QSV),                              avutil.PixelFormat(avcodec.AV_PIX_FMT_NV12), 1920, 1080, 64)          /*context_enc.SetHWFramesCtx(avutil.AVBufferRef(outs[0].AvBuffersinkGetHwFramesCtx()))        err = context_enc.AvcodecOpen2(codec_enc, nil)      if err < 0 {          log.Critical("AvcodecOpen2 failed.")          return      }*/        decLatency := latency.New("vaapi", "decode")      encLatency := latency.New("vaapi", "encode")        // filter      args := "video_size=1920x1080:pix_fmt=qsv:time_base=1/30:pixel_aspect=1/1"      //des  := "[in0] scale_qsv=w=960:h=540 [s0];color=black:r=30:size=1920x1080:sar=1/1, hwupload=extra_hw_frames=64,format=qsv [b0];[b0][s0] overlay_qsv=x=0:y=0"      des  := overlay16        graph := avfilter.AvfilterGraphAlloc()      if graph == nil {          log.Critical("AvfilterGraphAlloc Failed.")          return      }      //graph.SetNbThreads(8)        inputs  := avfilter.AvfilterInoutAlloc()      outputs := avfilter.AvfilterInoutAlloc()      if inputs == nil || outputs == nil {          log.Critical("AvfilterInoutAlloc Failed.")          return      }        defer avfilter.AvfilterInoutFree(inputs)      defer avfilter.AvfilterInoutFree(outputs)        var buffersrc *avfilter.Filter      var buffersink *avfilter.Filter      if false {          buffersrc  = avfilter.AvfilterGetByName("abuffer")          buffersink = avfilter.AvfilterGetByName("abuffersink")      } else {          buffersrc  = avfilter.AvfilterGetByName("buffer")          buffersink = avfilter.AvfilterGetByName("buffersink")      }        if buffersink == nil || buffersrc == nil {          log.Critical("AvfilterGetByName Failed.")          return      }        ret := graph.AvfilterGraphParse2(des, &inputs, &outputs)      if ret < 0 {          log.Critical("AvfilterInoutAlloc Failed des : ", avutil.ErrorFromCode(ret))          return      }        if qsv_device.Context() != nil {          for _, v := range graph.Filters() {              v.SetHWDeviceCtx(avutil.AVBufferRef(qsv_device.Context()))          }      }        var ins    []*avfilter.Context      var outs   []*avfilter.Context      var frames []*avutil.Frame        // inputs      index := 0        for cur := inputs; cur != nil; cur = cur.Next() {          //log.Debug("index :", index)          var in *avfilter.Context          //var args = "video_size=1280x720:pix_fmt=0:time_base=1/30:pixel_aspect=1/1"          inName := "in" + strconv.Itoa(index)          ret = avfilter.AvfilterGraphCreateFilter(&in, buffersrc, inName, args, 0, graph)          if ret < 0 {              log.Critical("AvfilterGraphCreateFilter Failed des : ", avutil.ErrorFromCode(ret))              return          }            par := avfilter.AvBuffersrcParametersAlloc()          if par == nil {              log.Critical("AvBuffersrcParametersAlloc Failed.")              return          }          par.SetHwFramesCtx(avutil.AVBufferRef(hw_frames_ref))          ret = in.AvBuffersrcParametersSet(par)          if ret < 0 {              log.Critical("AvBuffersrcParametersSet Failed.")              return          }          avutil.AvFreep(unsafe.Pointer(&par))            ins = append(ins, in)          ret = avfilter.AvfilterLink(ins[index], 0, cur.FilterContext(), cur.PadIdx())          if ret < 0 {              log.Critical("AvfilterLink Failed des : ", avutil.ErrorFromCode(ret))              return          }          index++      }        // outputs      index = 0      for cur := outputs; cur != nil; cur = cur.Next() {          var out *avfilter.Context          outName := "out" + strconv.Itoa(index)          ret = avfilter.AvfilterGraphCreateFilter(&out, buffersink, outName, "", 0, graph)          if ret < 0 {              log.Critical("AvfilterGraphCreateFilter Failed des : ", avutil.ErrorFromCode(ret))              return          }            outs = append(outs, out)          ret = avfilter.AvfilterLink(cur.FilterContext(), cur.PadIdx(), outs[index], 0)          if ret < 0 {              log.Critical("AvfilterLink Failed des : ", avutil.ErrorFromCode(ret))              return          }          index++            f := avutil.AvFrameAlloc()          if f == nil {              log.Critical("AvFrameAlloc failed.")              return          }          frames = append(frames, f)      }        ret = graph.AvfilterGraphConfig(0)      if ret < 0 {          log.Critical("AvfilterGraphConfig Failed des : ", avutil.ErrorFromCode(ret))          return      }        overlayLatency := latency.New("qsv", "overlay")        context_enc.SetHWFramesCtx(avutil.AVBufferRef(outs[0].AvBuffersinkGetHwFramesCtx()))        var dict *avutil.Dictionary = avutil.AvDictAlloc()      er := dict.AvDictSet("profile", "high", 0)      if er < 0 {          log.Critical("AvDictSet failed")          return      }      er = dict.AvDictSet("level", "52", 0)      if er < 0 {          log.Critical("AvDictSet failed")          return      }        fmt.Println(dict.AvDictCount())          err = context_enc.AvcodecOpen2(codec_enc, (**avcodec.Dictionary)(unsafe.Pointer(&dict)))      if err < 0 {          log.Critical("AvcodecOpen2 failed.")          return      }      fmt.Println(dict.AvDictCount())        data, erro := ioutil.ReadFile(input_h264)      if erro != nil {          log.Debug("File reading error", erro)          return      }      log.Debug("Open Success.")      l := len(data)      log.Debug("size of file:", l)        b := make([]byte, 4096 + 64)        file, erro := os.Create(output_h264)      if erro != nil {          log.Critical("Error Reading")      }      defer file.Close()        var pts int64 = 0      //var opts int64 = 0        //var frames [25]*avutil.Frame      started := false      sum := 0      for sum < l {          remain := 4096          for remain > 0 {              copy(b, data[sum:sum + 4096])              if !started {                  decLatency.Start()                  started = true              }              n := context.AvParserParse2(parserContext, pkt, b,                  remain, avcodec.AV_NOPTS_VALUE, avcodec.AV_NOPTS_VALUE, 0)              log.Debug("parser ", n, "bytes")                sum     = sum + n              remain  = remain - n;                //log.Trace("--------", dec.Packet().GetPacketSize())              if pkt.GetPacketSize() > 0 { // decode                  //fmt.Println(decLatency.End())                    ret := context.AvcodecSendPacket(pkt)                  if ret < 0 {                      log.Error("AvcodecSendPacket err ", avutil.ErrorFromCode(ret))                      continue                  }                    ret = context.AvcodecReceiveFrame((*avcodec.Frame)(unsafe.Pointer(frame)))                  if ret < 0 {                      log.Error("AvcodecReceiveFrame err ", avutil.ErrorFromCode(ret))                      continue                  }                    if ret == 0 {                      decLatency.End()                      fmt.Println(decLatency.End())                  } else {                      continue                  }                    overlayLatency.Start()                  pts++                  frame.SetPts(pts)                    var fs []*avutil.Frame                    for i := 0 ; i < 16 ; i++ {                      fs = append(fs, avutil.AvFrameClone(frame))                      ret = avfilter.AvBuffersrcAddFrame(ins[i], (*avfilter.Frame)(unsafe.Pointer(fs[i])))                      if ret < 0 {                          for i := 0 ; i < len(fs) ; i++ {                              avutil.AvFrameFree(fs[i])                          }                          fmt.Println("AvBuffersrcAddFrame error,", avutil.ErrorFromCode(ret))                          continue                      }                  }                  avutil.AvFrameUnref(frame)                  /*ret = avfilter.AvBuffersrcAddFrame(ins[0], (*avfilter.Frame)(unsafe.Pointer(frame)))                  if ret < 0 {                      fmt.Println("AvBuffersrcAddFrame error,", avutil.ErrorFromCode(ret))                      continue                  }*/                    ret = avfilter.AvBufferSinkGetFrame(outs[0], (*avfilter.Frame)(unsafe.Pointer(frames[0])))                  for i := 0 ; i < len(fs) ; i++ {                      avutil.AvFrameFree(fs[i])                  }                  if ret == avutil.AvErrorEOF || ret == avutil.AvErrorEAGAIN {                      log.Error(avutil.ErrorFromCode(ret))                      continue                  }                    if ret < 0 {                      log.Error("AvBufferSinkGetFrame Failed des : ", ret, avutil.ErrorFromCode(ret))                      continue                  }                    frame = frames[0]                    fmt.Println(overlayLatency.End())                    if ret == 0 { // encode                      started = false                        //log.Debug("dec-frame:",frame)                        encLatency.Start()                      ret = context_enc.AvcodecSendFrame((*avcodec.Frame)(unsafe.Pointer(frame)))                      if ret < 0 {                          log.Trace("AvcodecSendFrame err ", avutil.ErrorFromCode(ret))                          continue                      }                        ret = context_enc.AvcodecReceivePacket(pkt_enc)                      if ret < 0 {                          log.Trace("AvcodecReceivePacket err ", avutil.ErrorFromCode(ret))                          continue                      }                      if ret == 0 {                          data0 := pkt_enc.Data()                          buf := make([]byte, pkt_enc.GetPacketSize())                          start := uintptr(unsafe.Pointer(data0))                          for i := 0; i < pkt_enc.GetPacketSize(); i++ {                              elem := *(*uint8)(unsafe.Pointer(start + uintptr(i)))                              buf[i] = elem                          }                            file.Write(buf)                          //encLatency.End()                          fmt.Println(encLatency.End())                      }                        avutil.AvFrameUnref(frame)                      //avutil.AvFrameUnref(oframe)                  }              }          }      }  }        func filter_inpute_args() {      //decoder      qsv_device  := Create("/dev/dri/renderD128")        pkt            := avcodec.AvPacketAlloc()      if pkt == nil {          log.Critical("AvPacketAlloc failed.")          return      }        codec        := avcodec.AvcodecFindDecoderByName("h264_qsv")      if codec == nil {          log.Critical("AvcodecFindDecoderByName failed.")          return      }        context        := codec.AvcodecAllocContext3()      if context == nil {          log.Critical("AvcodecAllocContext3 failed.")          return      }        parserContext  := avcodec.AvParserInit(int(avcodec.CodecId(avcodec.AV_CODEC_ID_H264)))      if parserContext == nil {          log.Critical("AvParserInit failed.")          return      }        frame          := avutil.AvFrameAlloc()      if frame == nil {          log.Critical("AvFrameAlloc failed.")          return      }        context.SetHWDeviceCtx(avutil.AVBufferRef(qsv_device.Context()))      err := context.AvcodecOpen2(codec, nil)      if err < 0 {          log.Critical("AvcodecOpen2 failed.")          return      }      context.SetDefaultQsvGetFormat()        // encoder      codec_enc          := avcodec.AvcodecFindEncoderByName("h264_qsv")      if codec_enc == nil {          log.Critical("AvcodecFindEncoderByName failed.")      }        pkt_enc            := avcodec.AvPacketAlloc()      if pkt_enc == nil {          log.Critical("AvPacketAlloc failed.")      }        context_enc        := codec_enc.AvcodecAllocContext3()      if context_enc == nil {          log.Critical("AvcodecAllocContext3 failed.")      }        context_enc.SetVideoEncodeParams(2000000, 1920, 1080,          avcodec.AV_PIX_FMT_QSV,          false, 10)      context_enc.SetTimebase(1, 30)        hw_frames_ref := qsv_device.GetAnAllocHwframeCtxRef(                                  avutil.PixelFormat(avcodec.AV_PIX_FMT_QSV),                                  avutil.PixelFormat(avcodec.AV_PIX_FMT_NV12), 1920, 1080, 64)        pkt1           := avcodec.AvPacketAlloc()      if pkt == nil {          log.Critical("AvPacketAlloc failed.")          return      }        codec1       := avcodec.AvcodecFindDecoderByName("h264_qsv")      if codec == nil {          log.Critical("AvcodecFindDecoderByName failed.")          return      }        context1       := codec.AvcodecAllocContext3()      if context1 == nil {          log.Critical("AvcodecAllocContext3 failed.")          return      }        parserContext1  := avcodec.AvParserInit(int(avcodec.CodecId(avcodec.AV_CODEC_ID_H264)))      if parserContext1 == nil {          log.Critical("AvParserInit failed.")          return      }        frame1         := avutil.AvFrameAlloc()      if frame1 == nil {          log.Critical("AvFrameAlloc failed.")          return      }        context1.SetHWDeviceCtx(avutil.AVBufferRef(qsv_device.Context()))      err = context1.AvcodecOpen2(codec1, nil)      if err < 0 {          log.Critical("AvcodecOpen2 failed.")          return      }      context1.SetDefaultQsvGetFormat()          hw_frames_ref1 := qsv_device.GetAnAllocHwframeCtxRef(                              avutil.PixelFormat(avcodec.AV_PIX_FMT_QSV),                              avutil.PixelFormat(avcodec.AV_PIX_FMT_NV12), 1280, 720, 64)      /*context_enc.SetHWFramesCtx(avutil.AVBufferRef(outs[0].AvBuffersinkGetHwFramesCtx()))        err = context_enc.AvcodecOpen2(codec_enc, nil)      if err < 0 {          log.Critical("AvcodecOpen2 failed.")          return      }*/        decLatency := latency.New("vaapi", "decode")      encLatency := latency.New("vaapi", "encode")        // filter      args := "video_size=1920x1080:pix_fmt=qsv:time_base=1/30:pixel_aspect=1/1"      //des  := "[in0] scale_qsv=w=960:h=540 [s0];color=black:r=30:size=1920x1080:sar=1/1, hwupload=extra_hw_frames=64,format=qsv [b0];[b0][s0] overlay_qsv=x=0:y=0"      des  := overlay16        graph := avfilter.AvfilterGraphAlloc()      if graph == nil {          log.Critical("AvfilterGraphAlloc Failed.")          return      }      //graph.SetNbThreads(8)        inputs  := avfilter.AvfilterInoutAlloc()      outputs := avfilter.AvfilterInoutAlloc()      if inputs == nil || outputs == nil {          log.Critical("AvfilterInoutAlloc Failed.")          return      }        defer avfilter.AvfilterInoutFree(inputs)      defer avfilter.AvfilterInoutFree(outputs)        var buffersrc *avfilter.Filter      var buffersink *avfilter.Filter      if false {          buffersrc  = avfilter.AvfilterGetByName("abuffer")          buffersink = avfilter.AvfilterGetByName("abuffersink")      } else {          buffersrc  = avfilter.AvfilterGetByName("buffer")          buffersink = avfilter.AvfilterGetByName("buffersink")      }        if buffersink == nil || buffersrc == nil {          log.Critical("AvfilterGetByName Failed.")          return      }        ret := graph.AvfilterGraphParse2(des, &inputs, &outputs)      if ret < 0 {          log.Critical("AvfilterInoutAlloc Failed des : ", avutil.ErrorFromCode(ret))          return      }        if qsv_device.Context() != nil {          for _, v := range graph.Filters() {              v.SetHWDeviceCtx(avutil.AVBufferRef(qsv_device.Context()))          }      }        var ins    []*avfilter.Context      var outs   []*avfilter.Context      var frames []*avutil.Frame        // inputs      index := 0        for cur := inputs; cur != nil; cur = cur.Next() {          //log.Debug("index :", index)          var in *avfilter.Context          //var args = "video_size=1280x720:pix_fmt=0:time_base=1/30:pixel_aspect=1/1"          inName := "in" + strconv.Itoa(index)          ret = avfilter.AvfilterGraphCreateFilter(&in, buffersrc, inName, args, 0, graph)          if ret < 0 {              log.Critical("AvfilterGraphCreateFilter Failed des : ", avutil.ErrorFromCode(ret))              return          }            par := avfilter.AvBuffersrcParametersAlloc()          if par == nil {              log.Critical("AvBuffersrcParametersAlloc Failed.")              return          }          par.SetHwFramesCtx(avutil.AVBufferRef(hw_frames_ref))          ret = in.AvBuffersrcParametersSet(par)          if ret < 0 {              log.Critical("AvBuffersrcParametersSet Failed.")              return          }          avutil.AvFreep(unsafe.Pointer(&par))            ins = append(ins, in)          ret = avfilter.AvfilterLink(ins[index], 0, cur.FilterContext(), cur.PadIdx())          if ret < 0 {              log.Critical("AvfilterLink Failed des : ", avutil.ErrorFromCode(ret))              return          }          index++      }        // outputs      index = 0      for cur := outputs; cur != nil; cur = cur.Next() {          var out *avfilter.Context          outName := "out" + strconv.Itoa(index)          ret = avfilter.AvfilterGraphCreateFilter(&out, buffersink, outName, "", 0, graph)          if ret < 0 {              log.Critical("AvfilterGraphCreateFilter Failed des : ", avutil.ErrorFromCode(ret))              return          }            outs = append(outs, out)          ret = avfilter.AvfilterLink(cur.FilterContext(), cur.PadIdx(), outs[index], 0)          if ret < 0 {              log.Critical("AvfilterLink Failed des : ", avutil.ErrorFromCode(ret))              return          }          index++            f := avutil.AvFrameAlloc()          if f == nil {              log.Critical("AvFrameAlloc failed.")              return          }          frames = append(frames, f)      }        ret = graph.AvfilterGraphConfig(0)      if ret < 0 {          log.Critical("AvfilterGraphConfig Failed des : ", avutil.ErrorFromCode(ret))          return      }        overlayLatency := latency.New("qsv", "overlay")        context_enc.SetHWFramesCtx(avutil.AVBufferRef(outs[0].AvBuffersinkGetHwFramesCtx()))        var dict *avutil.Dictionary = avutil.AvDictAlloc()      er := dict.AvDictSet("profile", "high", 0)      if er < 0 {          log.Critical("AvDictSet failed")          return      }      er = dict.AvDictSet("level", "52", 0)      if er < 0 {          log.Critical("AvDictSet failed")          return      }        fmt.Println(dict.AvDictCount())          err = context_enc.AvcodecOpen2(codec_enc, (**avcodec.Dictionary)(unsafe.Pointer(&dict)))      if err < 0 {          log.Critical("AvcodecOpen2 failed.")          return      }      fmt.Println(dict.AvDictCount())        data, erro := ioutil.ReadFile(input_h264)      if erro != nil {          log.Debug("File reading error", erro)          return      }      log.Debug("Open Success.")      l := len(data)      log.Debug("size of file:", l)        b := make([]byte, 4096 + 64)        file, erro := os.Create(output_h264)      if erro != nil {          log.Critical("Error Reading")      }      defer file.Close()        var pts int64 = 0      //var opts int64 = 0        //var frames [25]*avutil.Frame      started := false      sum := 0      for sum < l {          remain := 4096          for remain > 0 {              copy(b, data[sum:sum + 4096])              if !started {                  decLatency.Start()                  started = true              }              n := context.AvParserParse2(parserContext, pkt, b,                  remain, avcodec.AV_NOPTS_VALUE, avcodec.AV_NOPTS_VALUE, 0)              log.Debug("parser ", n, "bytes")                sum     = sum + n              remain  = remain - n;                //log.Trace("--------", dec.Packet().GetPacketSize())              if pkt.GetPacketSize() > 0 { // decode                  //fmt.Println(decLatency.End())                    ret := context.AvcodecSendPacket(pkt)                  if ret < 0 {                      log.Error("AvcodecSendPacket err ", avutil.ErrorFromCode(ret))                      continue                  }                    ret = context.AvcodecReceiveFrame((*avcodec.Frame)(unsafe.Pointer(frame)))                  if ret < 0 {                      log.Error("AvcodecReceiveFrame err ", avutil.ErrorFromCode(ret))                      continue                  }                    if ret == 0 {                      decLatency.End()                      fmt.Println(decLatency.End())                  } else {                      continue                  }                    overlayLatency.Start()                  pts++                  frame.SetPts(pts)                    var fs []*avutil.Frame                    for i := 0 ; i < 16 ; i++ {                      fs = append(fs, avutil.AvFrameClone(frame))                      ret = avfilter.AvBuffersrcAddFrame(ins[i], (*avfilter.Frame)(unsafe.Pointer(fs[i])))                      if ret < 0 {                          for i := 0 ; i < len(fs) ; i++ {                              avutil.AvFrameFree(fs[i])                          }                          fmt.Println("AvBuffersrcAddFrame error,", avutil.ErrorFromCode(ret))                          continue                      }                  }                  avutil.AvFrameUnref(frame)                  /*ret = avfilter.AvBuffersrcAddFrame(ins[0], (*avfilter.Frame)(unsafe.Pointer(frame)))                  if ret < 0 {                      fmt.Println("AvBuffersrcAddFrame error,", avutil.ErrorFromCode(ret))                      continue                  }*/                    ret = avfilter.AvBufferSinkGetFrame(outs[0], (*avfilter.Frame)(unsafe.Pointer(frames[0])))                  for i := 0 ; i < len(fs) ; i++ {                      avutil.AvFrameFree(fs[i])                  }                  if ret == avutil.AvErrorEOF || ret == avutil.AvErrorEAGAIN {                      log.Error(avutil.ErrorFromCode(ret))                      continue                  }                    if ret < 0 {                      log.Error("AvBufferSinkGetFrame Failed des : ", ret, avutil.ErrorFromCode(ret))                      continue                  }                    frame = frames[0]                    fmt.Println(overlayLatency.End())                    if ret == 0 { // encode                      started = false                        //log.Debug("dec-frame:",frame)                        encLatency.Start()                      ret = context_enc.AvcodecSendFrame((*avcodec.Frame)(unsafe.Pointer(frame)))                      if ret < 0 {                          log.Trace("AvcodecSendFrame err ", avutil.ErrorFromCode(ret))                          continue                      }                        ret = context_enc.AvcodecReceivePacket(pkt_enc)                      if ret < 0 {                          log.Trace("AvcodecReceivePacket err ", avutil.ErrorFromCode(ret))                          continue                      }                      if ret == 0 {                          data0 := pkt_enc.Data()                          buf := make([]byte, pkt_enc.GetPacketSize())                          start := uintptr(unsafe.Pointer(data0))                          for i := 0; i < pkt_enc.GetPacketSize(); i++ {                              elem := *(*uint8)(unsafe.Pointer(start + uintptr(i)))                              buf[i] = elem                          }                            file.Write(buf)                          //encLatency.End()                          fmt.Println(encLatency.End())                      }                        avutil.AvFrameUnref(frame)                      //avutil.AvFrameUnref(oframe)                  }              }          }      }  }    func main() {      avutil.AvLogSetLevel(avutil.AV_LOG_TRACE)      //fmt.Println(filter.P1080InTestOverlay16)      //fmt.Println(avcodec.AV_PIX_FMT_QSV)      //fmt.Println(avcodec.AV_PIX_FMT_NV12)      //fmt.Println(avutil.ErrorFromCode(-15))        decode_overlay_encode()        time.Sleep(time.Hour)      //oclOpen()  }

参考 : Wiki/QuickSync