/* * Copyright (C) 2018 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICE3SESSION_H #define ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICE3SESSION_H #include #include #include #include #include #include #include #include #include #include #include #include "CameraMetadata.h" #include "HandleImporter.h" #include "Exif.h" #include "utils/KeyedVector.h" #include "utils/Mutex.h" #include "utils/Thread.h" #include "android-base/unique_fd.h" #include "ExternalCameraUtils.h" namespace android { namespace hardware { namespace camera { namespace device { namespace V3_4 { namespace implementation { using ::android::hardware::camera::device::V3_2::BufferCache; using ::android::hardware::camera::device::V3_2::BufferStatus; using ::android::hardware::camera::device::V3_2::CameraMetadata; using ::android::hardware::camera::device::V3_2::CaptureRequest; using ::android::hardware::camera::device::V3_2::CaptureResult; using ::android::hardware::camera::device::V3_2::ErrorCode; using ::android::hardware::camera::device::V3_2::ICameraDeviceCallback; using ::android::hardware::camera::device::V3_2::MsgType; using ::android::hardware::camera::device::V3_2::NotifyMsg; using ::android::hardware::camera::device::V3_2::RequestTemplate; using ::android::hardware::camera::device::V3_2::Stream; using ::android::hardware::camera::device::V3_4::StreamConfiguration; using ::android::hardware::camera::device::V3_2::StreamConfigurationMode; using ::android::hardware::camera::device::V3_2::StreamRotation; using ::android::hardware::camera::device::V3_2::StreamType; using ::android::hardware::camera::device::V3_2::DataspaceFlags; using ::android::hardware::camera::device::V3_2::CameraBlob; using ::android::hardware::camera::device::V3_2::CameraBlobId; using ::android::hardware::camera::device::V3_4::HalStreamConfiguration; using ::android::hardware::camera::device::V3_4::ICameraDeviceSession; using ::android::hardware::camera::common::V1_0::Status; using ::android::hardware::camera::common::V1_0::helper::HandleImporter; using ::android::hardware::camera::common::V1_0::helper::ExifUtils; using ::android::hardware::camera::external::common::ExternalCameraConfig; using ::android::hardware::camera::external::common::Size; using ::android::hardware::camera::external::common::SizeHasher; using ::android::hardware::graphics::common::V1_0::BufferUsage; using ::android::hardware::graphics::common::V1_0::Dataspace; using ::android::hardware::graphics::common::V1_0::PixelFormat; using ::android::hardware::kSynchronizedReadWrite; using ::android::hardware::MessageQueue; using ::android::hardware::MQDescriptorSync; using ::android::hardware::Return; using ::android::hardware::Void; using ::android::hardware::hidl_vec; using ::android::hardware::hidl_string; using ::android::sp; using ::android::Mutex; using ::android::base::unique_fd; struct ExternalCameraDeviceSession : public virtual RefBase { ExternalCameraDeviceSession(const sp&, const ExternalCameraConfig& cfg, const std::vector& sortedFormats, const CroppingType& croppingType, const common::V1_0::helper::CameraMetadata& chars, const std::string& cameraId, unique_fd v4l2Fd); virtual ~ExternalCameraDeviceSession(); // Call by CameraDevice to dump active device states void dumpState(const native_handle_t*); // Caller must use this method to check if CameraDeviceSession ctor failed bool isInitFailed() { return mInitFail; } bool isClosed(); // Retrieve the HIDL interface, split into its own class to avoid inheritance issues when // dealing with minor version revs and simultaneous implementation and interface inheritance virtual sp getInterface() { return new TrampolineSessionInterface_3_4(this); } static const int kMaxProcessedStream = 2; static const int kMaxStallStream = 1; static const uint32_t kMaxBytesPerPixel = 2; protected: // Methods from ::android::hardware::camera::device::V3_2::ICameraDeviceSession follow Return constructDefaultRequestSettings( RequestTemplate, ICameraDeviceSession::constructDefaultRequestSettings_cb _hidl_cb); Return configureStreams( const V3_2::StreamConfiguration&, ICameraDeviceSession::configureStreams_cb); Return getCaptureRequestMetadataQueue( ICameraDeviceSession::getCaptureRequestMetadataQueue_cb); Return getCaptureResultMetadataQueue( ICameraDeviceSession::getCaptureResultMetadataQueue_cb); Return processCaptureRequest( const hidl_vec&, const hidl_vec&, ICameraDeviceSession::processCaptureRequest_cb); Return flush(); Return close(); Return configureStreams_3_3( const V3_2::StreamConfiguration&, ICameraDeviceSession::configureStreams_3_3_cb); Return configureStreams_3_4( const V3_4::StreamConfiguration& requestedConfiguration, ICameraDeviceSession::configureStreams_3_4_cb _hidl_cb); Return processCaptureRequest_3_4( const hidl_vec& requests, const hidl_vec& cachesToRemove, ICameraDeviceSession::processCaptureRequest_3_4_cb _hidl_cb); protected: struct HalStreamBuffer { int32_t streamId; uint64_t bufferId; uint32_t width; uint32_t height; PixelFormat format; V3_2::BufferUsageFlags usage; buffer_handle_t* bufPtr; int acquireFence; bool fenceTimeout; }; struct HalRequest { uint32_t frameNumber; common::V1_0::helper::CameraMetadata setting; sp frameIn; nsecs_t shutterTs; std::vector buffers; }; Status constructDefaultRequestSettingsRaw(RequestTemplate type, V3_2::CameraMetadata *outMetadata); bool initialize(); Status initStatus() const; status_t initDefaultRequests(); status_t fillCaptureResult(common::V1_0::helper::CameraMetadata& md, nsecs_t timestamp); Status configureStreams(const V3_2::StreamConfiguration&, V3_3::HalStreamConfiguration* out); // fps = 0.0 means default, which is // slowest fps that is at least 30, or fastest fps if 30 is not supported int configureV4l2StreamLocked(const SupportedV4L2Format& fmt, double fps = 0.0); int v4l2StreamOffLocked(); int setV4l2FpsLocked(double fps); // TODO: change to unique_ptr for better tracking sp dequeueV4l2FrameLocked(/*out*/nsecs_t* shutterTs); // Called with mLock hold void enqueueV4l2Frame(const sp&); // Check if input Stream is one of supported stream setting on this device bool isSupported(const Stream&); // Validate and import request's output buffers and acquire fence Status importRequest( const CaptureRequest& request, hidl_vec& allBufPtrs, hidl_vec& allFences); static void cleanupInflightFences( hidl_vec& allFences, size_t numFences); void cleanupBuffersLocked(int id); void updateBufferCaches(const hidl_vec& cachesToRemove); Status processOneCaptureRequest(const CaptureRequest& request); Status processCaptureResult(std::shared_ptr&); Status processCaptureRequestError(const std::shared_ptr&); void notifyShutter(uint32_t frameNumber, nsecs_t shutterTs); void notifyError(uint32_t frameNumber, int32_t streamId, ErrorCode ec); void invokeProcessCaptureResultCallback( hidl_vec &results, bool tryWriteFmq); static void freeReleaseFences(hidl_vec&); Size getMaxJpegResolution() const; Size getMaxThumbResolution() const; ssize_t getJpegBufferSize(uint32_t width, uint32_t height) const; int waitForV4L2BufferReturnLocked(std::unique_lock& lk); class OutputThread : public android::Thread { public: OutputThread(wp parent, CroppingType); ~OutputThread(); Status allocateIntermediateBuffers( const Size& v4lSize, const Size& thumbSize, const hidl_vec& streams); Status submitRequest(const std::shared_ptr&); void flush(); void dump(int fd); virtual bool threadLoop() override; private: static const uint32_t FLEX_YUV_GENERIC = static_cast('F') | static_cast('L') << 8 | static_cast('E') << 16 | static_cast('X') << 24; // returns FLEX_YUV_GENERIC for formats other than YV12/YU12/NV12/NV21 static uint32_t getFourCcFromLayout(const YCbCrLayout&); static int getCropRect( CroppingType ct, const Size& inSize, const Size& outSize, IMapper::Rect* out); static const int kFlushWaitTimeoutSec = 3; // 3 sec static const int kReqWaitTimeoutMs = 33; // 33ms static const int kReqWaitTimesMax = 90; // 33ms * 90 ~= 3 sec void waitForNextRequest(std::shared_ptr* out); void signalRequestDone(); int cropAndScaleLocked( sp& in, const Size& outSize, YCbCrLayout* out); int cropAndScaleThumbLocked( sp& in, const Size& outSize, YCbCrLayout* out); int formatConvertLocked(const YCbCrLayout& in, const YCbCrLayout& out, Size sz, uint32_t format); static int encodeJpegYU12(const Size &inSz, const YCbCrLayout& inLayout, int jpegQuality, const void *app1Buffer, size_t app1Size, void *out, size_t maxOutSize, size_t &actualCodeSize); int createJpegLocked(HalStreamBuffer &halBuf, const std::shared_ptr& req); const wp mParent; const CroppingType mCroppingType; mutable std::mutex mRequestListLock; // Protect acccess to mRequestList, // mProcessingRequest and mProcessingFrameNumer std::condition_variable mRequestCond; // signaled when a new request is submitted std::condition_variable mRequestDoneCond; // signaled when a request is done processing std::list> mRequestList; bool mProcessingRequest = false; uint32_t mProcessingFrameNumer = 0; // V4L2 frameIn // (MJPG decode)-> mYu12Frame // (Scale)-> mScaledYu12Frames // (Format convert) -> output gralloc frames mutable std::mutex mBufferLock; // Protect access to intermediate buffers sp mYu12Frame; sp mYu12ThumbFrame; std::unordered_map, SizeHasher> mIntermediateBuffers; std::unordered_map, SizeHasher> mScaledYu12Frames; YCbCrLayout mYu12FrameLayout; YCbCrLayout mYu12ThumbFrameLayout; }; // Protect (most of) HIDL interface methods from synchronized-entering mutable Mutex mInterfaceLock; mutable Mutex mLock; // Protect all private members except otherwise noted const sp mCallback; const ExternalCameraConfig& mCfg; const common::V1_0::helper::CameraMetadata mCameraCharacteristics; const std::vector mSupportedFormats; const CroppingType mCroppingType; const std::string& mCameraId; unique_fd mV4l2Fd; // device is closed either // - closed by user // - init failed // - camera disconnected bool mClosed = false; bool mInitFail = false; bool mFirstRequest = false; common::V1_0::helper::CameraMetadata mLatestReqSetting; bool mV4l2Streaming = false; SupportedV4L2Format mV4l2StreamingFmt; double mV4l2StreamingFps = 0.0; size_t mV4L2BufferCount = 0; static const int kBufferWaitTimeoutSec = 3; // TODO: handle long exposure (or not allowing) std::mutex mV4l2BufferLock; // protect the buffer count and condition below std::condition_variable mV4L2BufferReturned; size_t mNumDequeuedV4l2Buffers = 0; uint32_t mMaxV4L2BufferSize = 0; // Not protected by mLock (but might be used when mLock is locked) sp mOutputThread; // Stream ID -> Camera3Stream cache std::unordered_map mStreamMap; std::unordered_set mInflightFrames; // buffers currently circulating between HAL and camera service // key: bufferId sent via HIDL interface // value: imported buffer_handle_t // Buffer will be imported during processCaptureRequest and will be freed // when the its stream is deleted or camera device session is closed typedef std::unordered_map CirculatingBuffers; // Stream ID -> circulating buffers map std::map mCirculatingBuffers; bool mAfTrigger = false; static HandleImporter sHandleImporter; /* Beginning of members not changed after initialize() */ using RequestMetadataQueue = MessageQueue; std::unique_ptr mRequestMetadataQueue; using ResultMetadataQueue = MessageQueue; std::shared_ptr mResultMetadataQueue; // Protect against invokeProcessCaptureResultCallback() Mutex mProcessCaptureResultLock; std::unordered_map mDefaultRequests; const Size mMaxThumbResolution; const Size mMaxJpegResolution; /* End of members not changed after initialize() */ private: struct TrampolineSessionInterface_3_4 : public ICameraDeviceSession { TrampolineSessionInterface_3_4(sp parent) : mParent(parent) {} virtual Return constructDefaultRequestSettings( RequestTemplate type, V3_3::ICameraDeviceSession::constructDefaultRequestSettings_cb _hidl_cb) override { return mParent->constructDefaultRequestSettings(type, _hidl_cb); } virtual Return configureStreams( const V3_2::StreamConfiguration& requestedConfiguration, V3_3::ICameraDeviceSession::configureStreams_cb _hidl_cb) override { return mParent->configureStreams(requestedConfiguration, _hidl_cb); } virtual Return processCaptureRequest(const hidl_vec& requests, const hidl_vec& cachesToRemove, V3_3::ICameraDeviceSession::processCaptureRequest_cb _hidl_cb) override { return mParent->processCaptureRequest(requests, cachesToRemove, _hidl_cb); } virtual Return getCaptureRequestMetadataQueue( V3_3::ICameraDeviceSession::getCaptureRequestMetadataQueue_cb _hidl_cb) override { return mParent->getCaptureRequestMetadataQueue(_hidl_cb); } virtual Return getCaptureResultMetadataQueue( V3_3::ICameraDeviceSession::getCaptureResultMetadataQueue_cb _hidl_cb) override { return mParent->getCaptureResultMetadataQueue(_hidl_cb); } virtual Return flush() override { return mParent->flush(); } virtual Return close() override { return mParent->close(); } virtual Return configureStreams_3_3( const V3_2::StreamConfiguration& requestedConfiguration, configureStreams_3_3_cb _hidl_cb) override { return mParent->configureStreams_3_3(requestedConfiguration, _hidl_cb); } virtual Return configureStreams_3_4( const V3_4::StreamConfiguration& requestedConfiguration, configureStreams_3_4_cb _hidl_cb) override { return mParent->configureStreams_3_4(requestedConfiguration, _hidl_cb); } virtual Return processCaptureRequest_3_4(const hidl_vec& requests, const hidl_vec& cachesToRemove, ICameraDeviceSession::processCaptureRequest_3_4_cb _hidl_cb) override { return mParent->processCaptureRequest_3_4(requests, cachesToRemove, _hidl_cb); } private: sp mParent; }; }; } // namespace implementation } // namespace V3_4 } // namespace device } // namespace camera } // namespace hardware } // namespace android #endif // ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICE3SESSION_H