In explanation openCamera Module of Android Camera Principle (2) The Camera Device Callbacks callback was mentioned in the article, but it was not expanded in detail at that time. In this article, we will explain it in detail.
Camera Device Callbacks generation process:
Summary of Android Camera Interprocess Communication Classes ICamera Device Callbacks. Aidl summarizes the generation process of Camera Device Callbacks in detail.
The callback interface in frameworks/av/camera/ndk/impl/ACameraDevice.h is as follows. This Camera Device Callbacks is set to the camera service side when the Camera is open. If the HAL layer has a Camera response, it will call the Service Callback interface in ACameraDevice. h to implement the callback. We need to figure out the scenarios in which these callbacks are triggered from the process, and understand these before we really understand the process of camera capture.

    // Callbacks from camera service
    class ServiceCallback : public hardware::camera2::BnCameraDeviceCallbacks {
      public:
        explicit ServiceCallback(CameraDevice* device) : mDevice(device) {}
        binder::Status onDeviceError(int32_t errorCode,
                           const CaptureResultExtras& resultExtras) override;
        binder::Status onDeviceIdle() override;
        binder::Status onCaptureStarted(const CaptureResultExtras& resultExtras,
                              int64_t timestamp) override;
        binder::Status onResultReceived(const CameraMetadata& metadata,
                              const CaptureResultExtras& resultExtras,
                              const std::vector<PhysicalCaptureResultInfo>& physicalResultInfos) override;
        binder::Status onPrepared(int streamId) override;
        binder::Status onRequestQueueEmpty() override;
        binder::Status onRepeatingRequestError(int64_t lastFrameNumber,
                int32_t stoppedSequenceId) override;
      private:
        const wp<CameraDevice> mDevice;
    };

1.onDeviceError

onDeviceError callback process. jpg

Another place to call is Camera 3D evice:: notify Error - > which is passed from HAL. For information about whether the current device is normal, if there is a problem with the device, call back - > listener - > notify Error (error code, result Extras);

void Camera3Device::notifyError(const camera3_error_msg_t &msg,
        sp<NotificationListener> listener) {
    ATRACE_CALL();
    // Map camera HAL error codes to ICameraDeviceCallback error codes
    // Index into this with the HAL error code
    static const int32_t halErrorMap[CAMERA3_MSG_NUM_ERRORS] = {
        // 0 = Unused error code
        hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_INVALID_ERROR,
        // 1 = CAMERA3_MSG_ERROR_DEVICE
        hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE,
        // 2 = CAMERA3_MSG_ERROR_REQUEST
        hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST,
        // 3 = CAMERA3_MSG_ERROR_RESULT
        hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT,
        // 4 = CAMERA3_MSG_ERROR_BUFFER
        hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER
    };

    int32_t errorCode =
            ((msg.error_code >= 0) &&
                    (msg.error_code < CAMERA3_MSG_NUM_ERRORS)) ?
            halErrorMap[msg.error_code] :
            hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_INVALID_ERROR;

    int streamId = 0;
    if (msg.error_stream != NULL) {
        Camera3Stream *stream =
                Camera3Stream::cast(msg.error_stream);
        streamId = stream->getId();
    }
    ALOGV("Camera %s: %s: HAL error, frame %d, stream %d: %d",
            mId.string(), __FUNCTION__, msg.frame_number,
            streamId, msg.error_code);

    CaptureResultExtras resultExtras;
    switch (errorCode) {
        case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE:
            // SET_ERR calls notifyError
            SET_ERR("Camera HAL reported serious device error");
            break;
        case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST:
        case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT:
        case hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER:
            {
                Mutex::Autolock l(mInFlightLock);
                ssize_t idx = mInFlightMap.indexOfKey(msg.frame_number);
                if (idx >= 0) {
                    InFlightRequest &r = mInFlightMap.editValueAt(idx);
                    r.requestStatus = msg.error_code;
                    resultExtras = r.resultExtras;
                    if (hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT == errorCode
                            ||  hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST ==
                            errorCode) {
                        r.skipResultMetadata = true;
                    }
                    if (hardware::camera2::ICameraDeviceCallbacks::ERROR_CAMERA_RESULT ==
                            errorCode) {
                        // In case of missing result check whether the buffers
                        // returned. If they returned, then remove inflight
                        // request.
                        removeInFlightRequestIfReadyLocked(idx);
                    }
                } else {
                    resultExtras.frameNumber = msg.frame_number;
                    ALOGE("Camera %s: %s: cannot find in-flight request on "
                            "frame %" PRId64 " error", mId.string(), __FUNCTION__,
                            resultExtras.frameNumber);
                }
            }
            resultExtras.errorStreamId = streamId;
            if (listener != NULL) {
                listener->notifyError(errorCode, resultExtras);
            } else {
                ALOGE("Camera %s: %s: no listener available", mId.string(), __FUNCTION__);
            }
            break;
        default:
            // SET_ERR calls notifyError
            SET_ERR("Unknown error message from HAL: %d", msg.error_code);
            break;
    }
}

2.onDeviceIdle

onDeviceIdle callback process. jpg

After the openCamera executes, it starts calling the onDeviceIdle callback. Camera3D evice:: initializeCommonLocked

 

status_t Camera3Device::initializeCommonLocked() {
    /** Start up status tracker thread */
    mStatusTracker = new StatusTracker(this);
    status_t res = mStatusTracker->run(String8::format("C3Dev-%s-Status", mId.string()).string());
    if (res != OK) {
        SET_ERR_L("Unable to start status tracking thread: %s (%d)",
                strerror(-res), res);
        mInterface->close();
        mStatusTracker.clear();
        return res;
    }
//......
}

StatusTracker is a thread defined in C++ and is somewhat similar to the java layer. Thread execution run automatically calls threadLoop(), which should be well understood.

class StatusTracker: public Thread {
  public:
    explicit StatusTracker(wp<Camera3Device> parent);
    ~StatusTracker();
    virtual void requestExit();
  protected:
    virtual bool threadLoop();
}

Next, the threadLoop() will be executed all the time, and if the current mStateTransitions [indicating device status] are found in IDLE during execution, it will be called back to the upper level.

    // Notify parent for all intermediate transitions
    if (mStateTransitions.size() > 0 && parent.get()) {
        for (size_t i = 0; i < mStateTransitions.size(); i++) {
            bool idle = (mStateTransitions[i] == IDLE);
            ALOGV("Camera device is now %s", idle ? "idle" : "active");
            parent->notifyStatus(idle);
        }
    }

3.onCaptureStarted

This callback function indicates that camera device is ready to call camera to get capture frame data. The callback begins with the camera HAL layer getting the underlying camera device driver notification that the current device is ready. Notification function in Camera3D evice::notify

void Camera3Device::notify(const camera3_notify_msg *msg) {
    ATRACE_CALL();
    sp<NotificationListener> listener;
    {
        Mutex::Autolock l(mOutputLock);
        listener = mListener.promote();
    }

    if (msg == NULL) {
        SET_ERR("HAL sent NULL notify message!");
        return;
    }

    switch (msg->type) {
        case CAMERA3_MSG_ERROR: {
            notifyError(msg->message.error, listener);
            break;
        }
        case CAMERA3_MSG_SHUTTER: {
            notifyShutter(msg->message.shutter, listener);
            break;
        }
        default:
            SET_ERR("Unknown notify message from HAL: %d",
                    msg->type);
    }
}

void CameraDeviceClient::notifyShutter(const CaptureResultExtras& resultExtras,
        nsecs_t timestamp) {
    // Thread safe. Don't bother locking.
    sp<hardware::camera2::ICameraDeviceCallbacks> remoteCb = getRemoteCallback();
    if (remoteCb != 0) {
        remoteCb->onCaptureStarted(resultExtras, timestamp);
    }
    Camera2ClientBase::notifyShutter(resultExtras, timestamp);
}

4.onResultReceived

This function is very important when previewing. It means that the captured frame data is returned continuously, and camera device is consuming capture frame continuously.

template<typename TProviderPtr>
status_t CameraDeviceClient::initializeImpl(TProviderPtr providerPtr, const String8& monitorTags) {
//......
    String8 threadName;
    mFrameProcessor = new FrameProcessorBase(mDevice);
    threadName = String8::format("CDU-%s-FrameProc", mCameraIdStr.string());
    mFrameProcessor->run(threadName.string());

    mFrameProcessor->registerListener(FRAME_PROCESSOR_LISTENER_MIN_ID,
                                      FRAME_PROCESSOR_LISTENER_MAX_ID,
                                      /*listener*/this,
                                      /*sendPartials*/true);
//......
}

bool FrameProcessorBase::threadLoop() {
    status_t res;

    sp<CameraDeviceBase> device;
    {
        device = mDevice.promote();
        if (device == 0) return false;
    }

    res = device->waitForNextFrame(kWaitDuration);
    if (res == OK) {
        processNewFrames(device);
    } else if (res != TIMED_OUT) {
        ALOGE("FrameProcessorBase: Error waiting for new "
                "frames: %s (%d)", strerror(-res), res);
    }

    return true;
}

/** Device-related methods */
void CameraDeviceClient::onResultAvailable(const CaptureResult& result) {
    ATRACE_CALL();
    ALOGV("%s", __FUNCTION__);

    // Thread-safe. No lock necessary.
    sp<hardware::camera2::ICameraDeviceCallbacks> remoteCb = mRemoteCallback;
    if (remoteCb != NULL) {
        remoteCb->onResultReceived(result.mMetadata, result.mResultExtras,
                result.mPhysicalMetadatas);
    }
}

5.onPrepared

There are two interfaces in Camera CaptureSession: this function is mainly used to pre-allocate memory to surface, but to speed up preview display.

    public abstract void prepare(@NonNull Surface surface) throws CameraAccessException;

    public abstract void prepare(int maxCount, @NonNull Surface surface)
            throws CameraAccessException;

Camera Device Callbacks - > onPrepared callbacks are performed after Camera Capture Session - > prepares are executed. This callback will only be triggered if the HAL application for camera device memory allocation is successful.

6.onRequestQueueEmpty

The onRequestQueueEmpty callback is that the current camera device's non-preview stream queue is empty, and it's starting to prepare for the next image of capture, just an intermediate state. It's not very important here. I'll just list the calling process.

• Camera3Device::RequestThread::threadLoop

bool Camera3Device::RequestThread::threadLoop() {
//......
    // Wait for the next batch of requests.
    waitForNextRequestBatch();
//......
}

• Camera3Device::RequestThread::waitForNextRequestBatch

void Camera3Device::RequestThread::waitForNextRequestBatch() {
//......
    NextRequest nextRequest;
    nextRequest.captureRequest = waitForNextRequestLocked();
    if (nextRequest.captureRequest == nullptr) {
        return;
    }

    nextRequest.halRequest = camera3_capture_request_t();
    nextRequest.submitted = false;
    mNextRequests.add(nextRequest);

    // Wait for additional requests
    const size_t batchSize = nextRequest.captureRequest->mBatchSize;

    for (size_t i = 1; i < batchSize; i++) {
        NextRequest additionalRequest;
        additionalRequest.captureRequest = waitForNextRequestLocked();
        if (additionalRequest.captureRequest == nullptr) {
            break;
        }

        additionalRequest.halRequest = camera3_capture_request_t();
        additionalRequest.submitted = false;
        mNextRequests.add(additionalRequest);
    }
//......
}

• Camera3Device::RequestThread::waitForNextRequestLocked

sp<Camera3Device::CaptureRequest>
        Camera3Device::RequestThread::waitForNextRequestLocked() {
//......
    if (nextRequest == NULL) {
        // Don't have a repeating request already in hand, so queue
        // must have an entry now.
        RequestList::iterator firstRequest =
                mRequestQueue.begin();
        nextRequest = *firstRequest;
        mRequestQueue.erase(firstRequest);
        if (mRequestQueue.empty() && !nextRequest->mRepeating) {
            sp<NotificationListener> listener = mListener.promote();
            if (listener != NULL) {
                listener->notifyRequestQueueEmpty();
            }
        }
    }
//......
}

void CameraDeviceClient::notifyRequestQueueEmpty() {
    // Thread safe. Don't bother locking.
    sp<hardware::camera2::ICameraDeviceCallbacks> remoteCb = getRemoteCallback();
    if (remoteCb != 0) {
        remoteCb->onRequestQueueEmpty();
    }
}

7.onRepeatingRequestError

A RequestThread thread is defined in Camera3D evice. h to manage the connection between capture request and HAL device.

    /**
     * Thread for managing capture request submission to HAL device.
     */
    class RequestThread : public Thread {
//......
      protected:
        virtual bool threadLoop();
    }

RequestThread starts in Camera 3D evice:: initializeCommonLocked.

    /** Start up request queue thread */
    mRequestThread = new RequestThread(this, mStatusTracker, mInterface, sessionParamKeys);
    res = mRequestThread->run(String8::format("C3Dev-%s-ReqQueue", mId.string()).string());
    if (res != OK) {
        SET_ERR_L("Unable to start request queue thread: %s (%d)",
                strerror(-res), res);
        mInterface->close();
        mRequestThread.clear();
        return res;
    }

The thread core execution logic is in the threadLoop function - > bool Camera3D evice:: RequestThread:: threadLoop ()

bool Camera3Device::RequestThread::threadLoop() {
//......
    // Prepare a batch of HAL requests and output buffers.
    res = prepareHalRequests();
    if (res == TIMED_OUT) {
        // Not a fatal error if getting output buffers time out.
        cleanUpFailedRequests(/*sendRequestError*/ true);
        // Check if any stream is abandoned.
        checkAndStopRepeatingRequest();
        return true;
    } else if (res != OK) {
        cleanUpFailedRequests(/*sendRequestError*/ false);
        return false;
    }
//......
}

Request the device at the HAL layer, and if the request times out, the preview at this time cannot proceed. Execute checkAndStop RepeatingRequest ();

void Camera3Device::RequestThread::checkAndStopRepeatingRequest() {
//......
    if (listener != NULL && surfaceAbandoned) {
        listener->notifyRepeatingRequestError(lastFrameNumber);
    }
}

Finally, it calls CameraDeviceClient:: notify RepeatingRequestError and calls back directly to the upper layer to notify the developer that the current capture request request request request failed.

void CameraDeviceClient::notifyRepeatingRequestError(long lastFrameNumber) {
    sp<hardware::camera2::ICameraDeviceCallbacks> remoteCb = getRemoteCallback();

    if (remoteCb != 0) {
        remoteCb->onRepeatingRequestError(lastFrameNumber, mStreamingRequestId);
    }

    Mutex::Autolock idLock(mStreamingRequestIdLock);
    mStreamingRequestId = REQUEST_ID_NONE;
}