2021SC@SDUSC BRPC source code analysis (VII) SOCKET

Socket.cpp code analysis

int Socket::HandleEpollOut(SocketId id) {
    SocketUniquePtr s;
    if (Socket::AddressFailedAsWell(id, &s) < 0) {
        // Ignore recycled sockets
        return -1;
    }

    EpollOutRequest* req = dynamic_cast<EpollOutRequest*>(s->user());
    if (req != NULL) {
        return s->HandleEpollOutRequest(0, req);
    }
    
    s->_epollout_butex->fetch_add(1, butil::memory_order_relaxed);
    bthread::butex_wake_except(s->_epollout_butex, 0);  
    return 0;
}

Socket s may have been 'SetFailed' before being added to epoll. These sockets have no signal in 'SetFailed', so 'AddressFailedAsWell' is used to send signals to prevent deadlock.

int Socket::HandleEpollOutRequest(int error_code, EpollOutRequest* req) {
    if (SetFailed() != 0) {
        return -1;
    }
    GetGlobalEventDispatcher(req->fd).RemoveEpollOut(id(), req->fd, false);
    return req->on_epollout_event(req->fd, error_code, req->data);
}

• There is and only one thread successfully called 'Socket' of 'SetFailed'.
• When the reference value is 0, the req is destroyed.
• Call the user's callback.
• In the destructor of EpollOutRequest, delete the timer.

int Socket::KeepWriteIfConnected(int fd, int err, void* data) {
    WriteRequest* req = static_cast<WriteRequest*>(data);
    Socket* s = req->socket;
    if (err == 0 && s->ssl_state() == SSL_CONNECTING) {
        bthread_t th;
        google::protobuf::Closure* thrd_func = brpc::NewCallback(
            Socket::CheckConnectedAndKeepWrite, fd, err, data);
        if ((err = bthread_start_background(&th, &BTHREAD_ATTR_NORMAL,
                                            RunClosure, thrd_func)) == 0) {
            return 0;
        } else {
            PLOG(ERROR) << "Fail to start bthread";
        }
    }
    CheckConnectedAndKeepWrite(fd, err, data);
    return 0;
}

• Run ssl connect in the new bthread to avoid blocking the current bthread (and EventDispatcher)
• Non zero 'err' output "Fail to start bthread" occurs
• Check the connection

	req->data.swap(*data);
    req->next = WriteRequest::UNCONNECTED;
    req->id_wait = opt.id_wait;
    req->set_pipelined_count_and_user_message(
        opt.pipelined_count, DUMMY_USER_MESSAGE, opt.with_auth);
    return StartWrite(req, opt);

Set 'req - > next' to UNCONNECTED so that the KeepWrite thread will wait until it points to a valid WriteRequest or NULL.

int Socket::StartWrite(WriteRequest* req, const WriteOptions& opt) {
    WriteRequest* const prev_head =
        _write_head.exchange(req, butil::memory_order_release);
    if (prev_head != NULL) {
        req->next = prev_head;
        return 0;
    }

    int saved_errno = 0;
    bthread_t th;
    SocketUniquePtr ptr_for_keep_write;
    ssize_t nw = 0;

    req->next = NULL;
    
    int ret = ConnectIfNot(opt.abstime, req);
    if (ret < 0) {
        saved_errno = errno;
        SetFailed(errno, "Fail to connect %s directly: %m", description().c_str());
        goto FAIL_TO_WRITE;
    } else if (ret == 1) {
        return 0;
    }

    req->Setup(this);
    
    if (ssl_state() != SSL_OFF) {
        goto KEEPWRITE_IN_BACKGROUND;
    }
   
    if (_conn) {
        butil::IOBuf* data_arr[1] = { &req->data };
        nw = _conn->CutMessageIntoFileDescriptor(fd(), data_arr, 1);
    } else {
        nw = req->data.cut_into_file_descriptor(fd());
    }
    if (nw < 0) {
        if (errno != EAGAIN && errno != EOVERCROWDED) {
            saved_errno = errno;
            PLOG_IF(WARNING, errno != EPIPE) << "Fail to write into " << *this;
            SetFailed(saved_errno, "Fail to write into %s: %s", 
                      description().c_str(), berror(saved_errno));
            goto FAIL_TO_WRITE;
        }
    } else {
        AddOutputBytes(nw);
    }
    if (IsWriteComplete(req, true, NULL)) {
        ReturnSuccessfulWriteRequest(req);
        return 0;
    }

KEEPWRITE_IN_BACKGROUND:
    ReAddress(&ptr_for_keep_write);
    req->socket = ptr_for_keep_write.release();
    if (bthread_start_background(&th, &BTHREAD_ATTR_NORMAL,
                                 KeepWrite, req) != 0) {
        LOG(FATAL) << "Fail to start KeepWrite";
        KeepWrite(req);
    }
    return 0;

FAIL_TO_WRITE:
    ReleaseAllFailedWriteRequests(req);
    errno = saved_errno;
    return -1;
}

• Release the fence to ensure that the requested thread gets * req.
• When data is written to fd. The KeepWrite thread may continue to request the Spin protocol until the next req - > next becomes non unconnected. This process is not lockless.
• Set write permissions.
• Test whether to connect to remote_side().
• Connect and CallbackKeepWriteIfConnected. KeepWriteIfConnected will be called by 'req'
• Setup() is called after Connect, and Connect may call app_. connect.
• Writing to SSL may block the current bthread. SSL writes in the background.
• Write once in the calling thread. If the write is not completed, continue in the KeepWrite thread.
• If NW < 0, RTMP may return eovercrossed.
• EPIPE connects to the pool and requests a backup.
• 'SetFailed 'before returnfailedwriterequest (the on_reset function called by returnfailedwriterequest and called back in the id object), so that you can immediately know that the Socket is on_ Callback failed in reset function.

 while (true) {
        int rc = SSL_do_handshake(_ssl_session);
        if (rc == 1) {
            _ssl_state = SSL_CONNECTED;
            AddBIOBuffer(_ssl_session, fd, FLAGS_ssl_bio_buffer_size);
            return 0;
        }

        int ssl_error = SSL_get_error(_ssl_session, rc);
        switch (ssl_error) {
        case SSL_ERROR_WANT_READ:
#if defined(OS_LINUX)
            if (bthread_fd_wait(fd, EPOLLIN) != 0) {
#elif defined(OS_MACOSX)
            if (bthread_fd_wait(fd, EVFILT_READ) != 0) {
#endif
                return -1;
            }

        case SSL_ERROR_WANT_WRITE:
#if defined(OS_LINUX)
            if (bthread_fd_wait(fd, EPOLLOUT) != 0) {
#elif defined(OS_MACOSX)
            if (bthread_fd_wait(fd, EVFILT_WRITE) != 0) {
#endif
                return -1;
            }

Loop until the SSL handshake is complete. For SSL_ERROR_WANT_READ/WRITE, using bthread_fd_wait is used as a polling mechanism rather than an EventDispatcher, which may confuse the original event processing code.

SocketPool (pool)

In prev rev, socketpools can be partitioned into multiple subsocketpool s to reduce thread contention. Fragment keys are mixed with pthread ID to better maintain data locality.

inline void SocketPool::ReturnSocket(Socket* sock) {
    const int connection_pool_size = FLAGS_max_connection_pool_size;

    if (_numfree.fetch_add(1, butil::memory_order_relaxed) <
        connection_pool_size) {
        const SocketId sid = sock->id();
        BAIDU_SCOPED_LOCK(_mutex);
        _pool.push_back(sid);
    } else {
        _numfree.fetch_sub(1, butil::memory_order_relaxed);
        sock->SetFailed(EUNUSED, "Close unused pooled socket");
    }
    _numinflight.fetch_sub(1, butil::memory_order_relaxed);
}

Save the gflag that can be reloaded at any time. Check whether the pool is full. When it is full, cancel adding and close the Socket in the pool.

int Socket::ReturnToPool() {
    SharedPart* sp = _shared_part.exchange(NULL, butil::memory_order_acquire);
    if (sp == NULL) {
        LOG(ERROR) << "_shared_part is NULL";
        SetFailed(EINVAL, "_shared_part is NULL");
        return -1;
    }
    SocketPool* pool = sp->socket_pool.load(butil::memory_order_consume);
    if (pool == NULL) {
        LOG(ERROR) << "_shared_part->socket_pool is NULL";
        SetFailed(EINVAL, "_shared_part->socket_pool is NULL");
        sp->RemoveRefManually();
        return -1;
    }
    _connection_type_for_progressive_read = CONNECTION_TYPE_UNKNOWN;
    _controller_released_socket.store(false, butil::memory_order_relaxed);
    pool->ReturnSocket(this);
    sp->RemoveRefManually();
    return 0;
}

• Returns an error when sp and pool are NULL.
• sp and pool are reset before returning to the pool.
• sp is released after returning to pool because sp has sp independent pool.