/* hfile_libcurl.c -- libcurl backend for low-level file streams. Copyright (C) 2015-2017 Genome Research Ltd. Author: John Marshall Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include #include #include #include #include #include #ifndef _WIN32 # include #endif #include #include "hfile_internal.h" #ifdef ENABLE_PLUGINS #include "version.h" #endif #include "htslib/hts.h" // for hts_version() and hts_verbose #include "htslib/kstring.h" #include "htslib/khash.h" #include // Number of seconds to take off auth_token expiry, to allow for clock skew // and slow servers #define AUTH_REFRESH_EARLY_SECS 60 // Minimum number of bytes to skip when seeking forward. Seeks less than // this will just read the data and throw it away. The optimal value // depends on how long it takes to make a new connection compared // to how fast the data arrives. #define MIN_SEEK_FORWARD 1000000 typedef struct { char *path; char *token; time_t expiry; int failed; pthread_mutex_t lock; } auth_token; // For the authorization header cache KHASH_MAP_INIT_STR(auth_map, auth_token *) // Curl-compatible header linked list typedef struct { struct curl_slist *list; unsigned int num; unsigned int size; } hdrlist; typedef struct { hdrlist fixed; // List of headers supplied at hopen() hdrlist extra; // List of headers from callback hts_httphdr_callback callback; // Callback to get more headers void *callback_data; // Data to pass to callback auth_token *auth; // Authentication token int auth_hdr_num; // Location of auth_token in hdrlist extra // If -1, Authorization header is in fixed // -2, it came from the callback // -3, "auth_token_enabled", "false" // passed to hopen() } http_headers; typedef struct { hFILE base; CURL *easy; CURLM *multi; off_t file_size; struct { union { char *rd; const char *wr; } ptr; size_t len; } buffer; CURLcode final_result; // easy result code for finished transfers // Flags for communicating with libcurl callbacks: unsigned paused : 1; // callback tells us that it has paused transfer unsigned closing : 1; // informs callback that hclose() has been invoked unsigned finished : 1; // wait_perform() tells us transfer is complete unsigned perform_again : 1; unsigned is_read : 1; // Opened in read mode unsigned can_seek : 1; // Can (attempt to) seek on this handle unsigned is_recursive:1; // Opened by hfile_libcurl itself unsigned tried_seek : 1; // At least one seek has been attempted int nrunning; http_headers headers; off_t delayed_seek; // Location to seek to before reading off_t last_offset; // Location we're seeking from } hFILE_libcurl; static off_t libcurl_seek(hFILE *fpv, off_t offset, int whence); static int restart_from_position(hFILE_libcurl *fp, off_t pos); static int http_status_errno(int status) { if (status >= 500) switch (status) { case 501: return ENOSYS; case 503: return EBUSY; case 504: return ETIMEDOUT; default: return EIO; } else if (status >= 400) switch (status) { case 401: return EPERM; case 403: return EACCES; case 404: return ENOENT; case 405: return EROFS; case 407: return EPERM; case 408: return ETIMEDOUT; case 410: return ENOENT; default: return EINVAL; } else return 0; } static int easy_errno(CURL *easy, CURLcode err) { long lval; switch (err) { case CURLE_OK: return 0; case CURLE_UNSUPPORTED_PROTOCOL: case CURLE_URL_MALFORMAT: return EINVAL; #if LIBCURL_VERSION_NUM >= 0x071505 case CURLE_NOT_BUILT_IN: return ENOSYS; #endif case CURLE_COULDNT_RESOLVE_PROXY: case CURLE_COULDNT_RESOLVE_HOST: case CURLE_FTP_CANT_GET_HOST: return EDESTADDRREQ; // Lookup failure case CURLE_COULDNT_CONNECT: case CURLE_SEND_ERROR: case CURLE_RECV_ERROR: if (curl_easy_getinfo(easy, CURLINFO_OS_ERRNO, &lval) == CURLE_OK) return lval; else return ECONNABORTED; case CURLE_REMOTE_ACCESS_DENIED: case CURLE_LOGIN_DENIED: case CURLE_TFTP_PERM: return EACCES; case CURLE_PARTIAL_FILE: return EPIPE; case CURLE_HTTP_RETURNED_ERROR: if (curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &lval) == CURLE_OK) return http_status_errno(lval); else return EIO; case CURLE_OUT_OF_MEMORY: return ENOMEM; case CURLE_OPERATION_TIMEDOUT: return ETIMEDOUT; case CURLE_RANGE_ERROR: return ESPIPE; case CURLE_SSL_CONNECT_ERROR: // TODO return SSL error buffer messages return ECONNABORTED; case CURLE_FILE_COULDNT_READ_FILE: case CURLE_TFTP_NOTFOUND: return ENOENT; case CURLE_TOO_MANY_REDIRECTS: return ELOOP; case CURLE_FILESIZE_EXCEEDED: return EFBIG; case CURLE_REMOTE_DISK_FULL: return ENOSPC; case CURLE_REMOTE_FILE_EXISTS: return EEXIST; default: return EIO; } } static int multi_errno(CURLMcode errm) { switch (errm) { case CURLM_CALL_MULTI_PERFORM: case CURLM_OK: return 0; case CURLM_BAD_HANDLE: case CURLM_BAD_EASY_HANDLE: case CURLM_BAD_SOCKET: return EBADF; case CURLM_OUT_OF_MEMORY: return ENOMEM; default: return EIO; } } static struct { kstring_t useragent; CURLSH *share; char *auth_path; khash_t(auth_map) *auth_map; int allow_unencrypted_auth_header; pthread_mutex_t auth_lock; pthread_mutex_t share_lock; } curl = { { 0, 0, NULL }, NULL, NULL, NULL, 0, PTHREAD_MUTEX_INITIALIZER, PTHREAD_MUTEX_INITIALIZER }; static void share_lock(CURL *handle, curl_lock_data data, curl_lock_access access, void *userptr) { pthread_mutex_lock(&curl.share_lock); } static void share_unlock(CURL *handle, curl_lock_data data, void *userptr) { pthread_mutex_unlock(&curl.share_lock); } static void free_auth(auth_token *tok) { if (!tok) return; if (pthread_mutex_destroy(&tok->lock)) abort(); free(tok->path); free(tok->token); free(tok); } static void libcurl_exit() { if (curl_share_cleanup(curl.share) == CURLSHE_OK) curl.share = NULL; free(curl.useragent.s); curl.useragent.l = curl.useragent.m = 0; curl.useragent.s = NULL; free(curl.auth_path); curl.auth_path = NULL; if (curl.auth_map) { khiter_t i; for (i = kh_begin(curl.auth_map); i != kh_end(curl.auth_map); ++i) { if (kh_exist(curl.auth_map, i)) { free_auth(kh_value(curl.auth_map, i)); kh_key(curl.auth_map, i) = NULL; kh_value(curl.auth_map, i) = NULL; } } kh_destroy(auth_map, curl.auth_map); curl.auth_map = NULL; } curl_global_cleanup(); } static int append_header(hdrlist *hdrs, const char *data, int dup) { if (hdrs->num == hdrs->size) { unsigned int new_sz = hdrs->size ? hdrs->size * 2 : 4, i; struct curl_slist *new_list = realloc(hdrs->list, new_sz * sizeof(*new_list)); if (!new_list) return -1; hdrs->size = new_sz; hdrs->list = new_list; for (i = 1; i < hdrs->num; i++) hdrs->list[i-1].next = &hdrs->list[i]; } // Annoyingly, libcurl doesn't declare the char * as const... hdrs->list[hdrs->num].data = dup ? strdup(data) : (char *) data; if (!hdrs->list[hdrs->num].data) return -1; if (hdrs->num > 0) hdrs->list[hdrs->num - 1].next = &hdrs->list[hdrs->num]; hdrs->list[hdrs->num].next = NULL; hdrs->num++; return 0; } static void free_headers(hdrlist *hdrs, int completely) { unsigned int i; for (i = 0; i < hdrs->num; i++) { free(hdrs->list[i].data); hdrs->list[i].data = NULL; hdrs->list[i].next = NULL; } hdrs->num = 0; if (completely) { free(hdrs->list); hdrs->size = 0; hdrs->list = NULL; } } static struct curl_slist * get_header_list(hFILE_libcurl *fp) { if (fp->headers.fixed.num > 0) return &fp->headers.fixed.list[0]; if (fp->headers.extra.num > 0) return &fp->headers.extra.list[0]; return 0; } static inline int is_authorization(const char *hdr) { return (strncasecmp("authorization:", hdr, 14) == 0); } static int add_callback_headers(hFILE_libcurl *fp) { char **hdrs = NULL, **hdr; if (!fp->headers.callback) return 0; // Get the headers from the callback if (fp->headers.callback(fp->headers.callback_data, &hdrs) != 0) { return -1; } if (!hdrs) // No change return 0; // Remove any old callback headers if (fp->headers.fixed.num > 0) { // Unlink lists fp->headers.fixed.list[fp->headers.fixed.num - 1].next = NULL; } free_headers(&fp->headers.extra, 0); if (fp->headers.auth_hdr_num > 0 || fp->headers.auth_hdr_num == -2) fp->headers.auth_hdr_num = 0; // Just removed it... // Convert to libcurl-suitable form for (hdr = hdrs; *hdr; hdr++) { if (append_header(&fp->headers.extra, *hdr, 0) < 0) { goto cleanup; } if (is_authorization(*hdr) && !fp->headers.auth_hdr_num) fp->headers.auth_hdr_num = -2; } for (hdr = hdrs; *hdr; hdr++) *hdr = NULL; if (fp->headers.fixed.num > 0 && fp->headers.extra.num > 0) { // Relink lists fp->headers.fixed.list[fp->headers.fixed.num - 1].next = &fp->headers.extra.list[0]; } return 0; cleanup: while (hdr && *hdr) { free(*hdr); *hdr = NULL; } return -1; } /* * Read an OAUTH2-style Bearer access token (see * https://tools.ietf.org/html/rfc6750#section-4). * Returns 'v' for valid; 'i' for invalid (token missing or wrong sort); * '?' for a JSON parse error; 'm' if it runs out of memory. */ static int read_auth_json(auth_token *tok, hFILE *auth_fp) { hts_json_token *t = hts_json_alloc_token(); kstring_t str = {0, 0, NULL}; char *token = NULL, *type = NULL, *expiry = NULL; int ret = 'i'; if (!t) goto error; if ((ret = hts_json_fnext(auth_fp, t, &str)) != '{') goto error; while (hts_json_fnext(auth_fp, t, &str) != '}') { char *key; if (hts_json_token_type(t) != 's') { ret = '?'; goto error; } key = hts_json_token_str(t); if (!key) goto error; if (strcmp(key, "access_token") == 0) { if ((ret = hts_json_fnext(auth_fp, t, &str)) != 's') goto error; token = ks_release(&str); } else if (strcmp(key, "token_type") == 0) { if ((ret = hts_json_fnext(auth_fp, t, &str)) != 's') goto error; type = ks_release(&str); } else if (strcmp(key, "expires_in") == 0) { if ((ret = hts_json_fnext(auth_fp, t, &str)) != 'n') goto error; expiry = ks_release(&str); } else if (hts_json_fskip_value(auth_fp, '\0') != 'v') { ret = '?'; goto error; } } if (!token || (type && strcmp(type, "Bearer") != 0)) { ret = 'i'; goto error; } ret = 'm'; str.l = 0; if (kputs("Authorization: Bearer ", &str) < 0) goto error; if (kputs(token, &str) < 0) goto error; free(tok->token); tok->token = ks_release(&str); if (expiry) { long exp = strtol(expiry, NULL, 10); if (exp < 0) exp = 0; tok->expiry = time(NULL) + exp; } else { tok->expiry = 0; } ret = 'v'; error: free(token); free(type); free(expiry); free(str.s); hts_json_free_token(t); return ret; } static int read_auth_plain(auth_token *tok, hFILE *auth_fp) { kstring_t line = {0, 0, NULL}; kstring_t token = {0, 0, NULL}; const char *start, *end; if (kgetline(&line, (char * (*)(char *, int, void *)) hgets, auth_fp) < 0) goto error; if (kputc('\0', &line) < 0) goto error; for (start = line.s; *start && isspace_c(*start); start++) {} for (end = start; *end && !isspace_c(*end); end++) {} if (end > start) { if (kputs("Authorization: Bearer ", &token) < 0) goto error; if (kputsn(start, end - start, &token) < 0) goto error; } free(tok->token); tok->token = ks_release(&token); tok->expiry = 0; free(line.s); return 0; error: free(line.s); free(token.s); return -1; } static int renew_auth_token(auth_token *tok, int *changed) { hFILE *auth_fp = NULL; char buffer[16]; ssize_t len; *changed = 0; if (tok->expiry == 0 || time(NULL) + AUTH_REFRESH_EARLY_SECS < tok->expiry) return 0; // Still valid if (tok->failed) return -1; *changed = 1; auth_fp = hopen(tok->path, "rR"); if (!auth_fp) { // Not worried about missing files; other errors are bad. if (errno != ENOENT) goto fail; tok->expiry = 0; // Prevent retry free(tok->token); // Just in case it was set return 0; } len = hpeek(auth_fp, buffer, sizeof(buffer)); if (len < 0) goto fail; if (memchr(buffer, '{', len) != NULL) { if (read_auth_json(tok, auth_fp) != 'v') goto fail; } else { if (read_auth_plain(tok, auth_fp) < 0) goto fail; } return hclose(auth_fp) < 0 ? -1 : 0; fail: tok->failed = 1; if (auth_fp) hclose_abruptly(auth_fp); return -1; } static int add_auth_header(hFILE_libcurl *fp) { int changed = 0; if (fp->headers.auth_hdr_num < 0) return 0; // Have an Authorization header from open or header callback if (!fp->headers.auth) return 0; // Nothing to add pthread_mutex_lock(&fp->headers.auth->lock); if (renew_auth_token(fp->headers.auth, &changed) < 0) goto unlock_fail; if (!changed && fp->headers.auth_hdr_num > 0) { pthread_mutex_unlock(&fp->headers.auth->lock); return 0; } if (fp->headers.auth_hdr_num > 0) { // Had a previous header, so swap in the new one char *header = fp->headers.auth->token; char *header_copy = header ? strdup(header) : NULL; int idx = fp->headers.auth_hdr_num - 1; if (header && !header_copy) goto unlock_fail; if (header_copy) { free(fp->headers.extra.list[idx].data); fp->headers.extra.list[idx].data = header_copy; } else { unsigned int j; // More complicated case - need to get rid of the old header // and tidy up linked lists free(fp->headers.extra.list[idx].data); for (j = idx + 1; j < fp->headers.extra.num; j++) { fp->headers.extra.list[j - 1] = fp->headers.extra.list[j]; fp->headers.extra.list[j - 1].next = &fp->headers.extra.list[j]; } fp->headers.extra.num--; if (fp->headers.extra.num > 0) { fp->headers.extra.list[fp->headers.extra.num-1].next = NULL; } else if (fp->headers.fixed.num > 0) { fp->headers.fixed.list[fp->headers.fixed.num - 1].next = NULL; } fp->headers.auth_hdr_num = 0; } } else if (fp->headers.auth->token) { // Add new header and remember where it is if (append_header(&fp->headers.extra, fp->headers.auth->token, 1) < 0) { goto unlock_fail; } fp->headers.auth_hdr_num = fp->headers.extra.num; } pthread_mutex_unlock(&fp->headers.auth->lock); return 0; unlock_fail: pthread_mutex_unlock(&fp->headers.auth->lock); return -1; } static int get_auth_token(hFILE_libcurl *fp, const char *url) { const char *host = NULL, *p, *q; kstring_t name = {0, 0, NULL}; size_t host_len = 0; khiter_t idx; auth_token *tok = NULL; // Nothing to do if: // curl.auth_path has not been set // fp was made by hfile_libcurl (e.g. auth_path is a http:// url) // we already have an Authorization header if (!curl.auth_path || fp->is_recursive || fp->headers.auth_hdr_num != 0) return 0; // Insist on having a secure connection unless the user insists harder if (!curl.allow_unencrypted_auth_header && strncmp(url, "https://", 8) != 0) return 0; host = strstr(url, "://"); if (host) { host += 3; host_len = strcspn(host, "/"); } p = curl.auth_path; while ((q = strstr(p, "%h")) != NULL) { if (q - p > INT_MAX || host_len > INT_MAX) goto error; if (kputsn_(p, q - p, &name) < 0) goto error; if (kputsn_(host, host_len, &name) < 0) goto error; p = q + 2; } if (kputs(p, &name) < 0) goto error; pthread_mutex_lock(&curl.auth_lock); idx = kh_get(auth_map, curl.auth_map, name.s); if (idx < kh_end(curl.auth_map)) { tok = kh_value(curl.auth_map, idx); } else { tok = calloc(1, sizeof(*tok)); if (tok && pthread_mutex_init(&tok->lock, NULL) != 0) { free(tok); tok = NULL; } if (tok) { int ret = -1; tok->path = ks_release(&name); tok->token = NULL; tok->expiry = 1; // Force refresh idx = kh_put(auth_map, curl.auth_map, tok->path, &ret); if (ret < 0) { free_auth(tok); tok = NULL; } kh_value(curl.auth_map, idx) = tok; } } pthread_mutex_unlock(&curl.auth_lock); fp->headers.auth = tok; free(name.s); return add_auth_header(fp); error: free(name.s); return -1; } static void process_messages(hFILE_libcurl *fp) { CURLMsg *msg; int remaining; while ((msg = curl_multi_info_read(fp->multi, &remaining)) != NULL) { switch (msg->msg) { case CURLMSG_DONE: fp->finished = 1; fp->final_result = msg->data.result; break; default: break; } } } static int wait_perform(hFILE_libcurl *fp) { fd_set rd, wr, ex; int maxfd, nrunning; long timeout; CURLMcode errm; if (!fp->perform_again) { FD_ZERO(&rd); FD_ZERO(&wr); FD_ZERO(&ex); if (curl_multi_fdset(fp->multi, &rd, &wr, &ex, &maxfd) != CURLM_OK) maxfd = -1, timeout = 1000; else { if (curl_multi_timeout(fp->multi, &timeout) != CURLM_OK) timeout = 1000; else if (timeout < 0) { timeout = 10000; // as recommended by curl_multi_timeout(3) } } if (maxfd < 0 && timeout > 100) timeout = 100; // as recommended by curl_multi_fdset(3) if (timeout > 0) { struct timeval tval; tval.tv_sec = (timeout / 1000); tval.tv_usec = (timeout % 1000) * 1000; if (select(maxfd + 1, &rd, &wr, &ex, &tval) < 0) return -1; } } errm = curl_multi_perform(fp->multi, &nrunning); fp->perform_again = 0; if (errm == CURLM_CALL_MULTI_PERFORM) fp->perform_again = 1; else if (errm != CURLM_OK) { errno = multi_errno(errm); return -1; } if (nrunning < fp->nrunning) process_messages(fp); return 0; } static size_t recv_callback(char *ptr, size_t size, size_t nmemb, void *fpv) { hFILE_libcurl *fp = (hFILE_libcurl *) fpv; size_t n = size * nmemb; if (n > fp->buffer.len) { fp->paused = 1; return CURL_WRITEFUNC_PAUSE; } else if (n == 0) return 0; memcpy(fp->buffer.ptr.rd, ptr, n); fp->buffer.ptr.rd += n; fp->buffer.len -= n; return n; } static ssize_t libcurl_read(hFILE *fpv, void *bufferv, size_t nbytes) { hFILE_libcurl *fp = (hFILE_libcurl *) fpv; char *buffer = (char *) bufferv; off_t to_skip = -1; ssize_t got = 0; CURLcode err; if (fp->delayed_seek >= 0) { assert(fp->base.offset == fp->delayed_seek && fp->base.begin == fp->base.buffer && fp->base.end == fp->base.buffer); if (fp->last_offset >= 0 && fp->delayed_seek > fp->last_offset && fp->delayed_seek - fp->last_offset < MIN_SEEK_FORWARD) { // If not seeking far, just read the data and throw it away. This // is likely to be quicker than opening a new stream to_skip = fp->delayed_seek - fp->last_offset; } else { if (restart_from_position(fp, fp->delayed_seek) < 0) { return -1; } } fp->delayed_seek = -1; fp->last_offset = -1; } do { fp->buffer.ptr.rd = buffer; fp->buffer.len = nbytes; fp->paused = 0; err = curl_easy_pause(fp->easy, CURLPAUSE_CONT); if (err != CURLE_OK) { errno = easy_errno(fp->easy, err); return -1; } while (! fp->paused && ! fp->finished) if (wait_perform(fp) < 0) return -1; got = fp->buffer.ptr.rd - buffer; if (to_skip >= 0) { // Skipping over a small seek if (got < to_skip) { // Need to skip more data to_skip -= got; } else { got -= to_skip; if (got > 0) { // If enough was skipped, return the rest memmove(buffer, buffer + to_skip, got); to_skip = -1; } } } } while (to_skip >= 0 && ! fp->finished); fp->buffer.ptr.rd = NULL; fp->buffer.len = 0; if (fp->finished && fp->final_result != CURLE_OK) { errno = easy_errno(fp->easy, fp->final_result); return -1; } return got; } static size_t send_callback(char *ptr, size_t size, size_t nmemb, void *fpv) { hFILE_libcurl *fp = (hFILE_libcurl *) fpv; size_t n = size * nmemb; if (fp->buffer.len == 0) { // Send buffer is empty; normally pause, or signal EOF if we're closing if (fp->closing) return 0; else { fp->paused = 1; return CURL_READFUNC_PAUSE; } } if (n > fp->buffer.len) n = fp->buffer.len; memcpy(ptr, fp->buffer.ptr.wr, n); fp->buffer.ptr.wr += n; fp->buffer.len -= n; return n; } static ssize_t libcurl_write(hFILE *fpv, const void *bufferv, size_t nbytes) { hFILE_libcurl *fp = (hFILE_libcurl *) fpv; const char *buffer = (const char *) bufferv; CURLcode err; fp->buffer.ptr.wr = buffer; fp->buffer.len = nbytes; fp->paused = 0; err = curl_easy_pause(fp->easy, CURLPAUSE_CONT); if (err != CURLE_OK) { errno = easy_errno(fp->easy, err); return -1; } while (! fp->paused && ! fp->finished) if (wait_perform(fp) < 0) return -1; nbytes = fp->buffer.ptr.wr - buffer; fp->buffer.ptr.wr = NULL; fp->buffer.len = 0; if (fp->finished && fp->final_result != CURLE_OK) { errno = easy_errno(fp->easy, fp->final_result); return -1; } return nbytes; } static off_t libcurl_seek(hFILE *fpv, off_t offset, int whence) { hFILE_libcurl *fp = (hFILE_libcurl *) fpv; off_t origin, pos; if (!fp->is_read || !fp->can_seek) { // Cowardly refuse to seek when writing or a previous seek failed. errno = ESPIPE; return -1; } switch (whence) { case SEEK_SET: origin = 0; break; case SEEK_CUR: errno = ENOSYS; return -1; case SEEK_END: if (fp->file_size < 0) { errno = ESPIPE; return -1; } origin = fp->file_size; break; default: errno = EINVAL; return -1; } // Check 0 <= origin+offset < fp->file_size carefully, avoiding overflow if ((offset < 0)? origin + offset < 0 : (fp->file_size >= 0 && offset > fp->file_size - origin)) { errno = EINVAL; return -1; } pos = origin + offset; if (fp->tried_seek) { /* Seeking has worked at least once, so now we can delay doing the actual work until the next read. This avoids lots of pointless http or ftp reconnections if the caller does lots of seeks without any intervening reads. */ if (fp->delayed_seek < 0) { fp->last_offset = fp->base.offset + (fp->base.end - fp->base.buffer); } fp->delayed_seek = pos; return pos; } if (restart_from_position(fp, pos) < 0) { /* This value for errno may not be entirely true, but the caller may be able to carry on with the existing handle. */ errno = ESPIPE; return -1; } fp->tried_seek = 1; return pos; } static int restart_from_position(hFILE_libcurl *fp, off_t pos) { hFILE_libcurl temp_fp; CURLcode err; CURLMcode errm; int update_headers = 0; int save_errno = 0; // TODO If we seem to be doing random access, use CURLOPT_RANGE to do // limited reads (e.g. about a BAM block!) so seeking can reuse the // existing connection more often. // Get new headers from the callback (if defined). This changes the // headers in fp before it gets duplicated, but they should be have been // sent by now. if (fp->headers.callback) { if (add_callback_headers(fp) != 0) return -1; update_headers = 1; } if (fp->headers.auth_hdr_num > 0 && fp->headers.auth) { if (add_auth_header(fp) != 0) return -1; update_headers = 1; } if (update_headers) { struct curl_slist *list = get_header_list(fp); if (list) { err = curl_easy_setopt(fp->easy, CURLOPT_HTTPHEADER, list); if (err != CURLE_OK) { errno = easy_errno(fp->easy,err); return -1; } } } /* Duplicate the easy handle, and use CURLOPT_RESUME_FROM_LARGE to open a new request to the server, reading from the location that we want to seek to. If the new request works and returns the correct data, the original easy handle in *fp is closed and replaced with the new one. If not, we close the new handle and leave *fp unchanged. */ memcpy(&temp_fp, fp, sizeof(temp_fp)); temp_fp.buffer.len = 0; temp_fp.buffer.ptr.rd = NULL; temp_fp.easy = curl_easy_duphandle(fp->easy); if (!temp_fp.easy) goto early_error; err = curl_easy_setopt(temp_fp.easy, CURLOPT_RESUME_FROM_LARGE,(curl_off_t)pos); err |= curl_easy_setopt(temp_fp.easy, CURLOPT_PRIVATE, &temp_fp); err |= curl_easy_setopt(temp_fp.easy, CURLOPT_WRITEDATA, &temp_fp); if (err != CURLE_OK) { save_errno = easy_errno(temp_fp.easy, err); goto error; } temp_fp.buffer.len = 0; // Ensures we only read the response headers temp_fp.paused = temp_fp.finished = 0; // fp->multi and temp_fp.multi are the same. errm = curl_multi_add_handle(fp->multi, temp_fp.easy); if (errm != CURLM_OK) { save_errno = multi_errno(errm); goto error; } temp_fp.nrunning = ++fp->nrunning; err = curl_easy_pause(temp_fp.easy, CURLPAUSE_CONT); if (err != CURLE_OK) { save_errno = easy_errno(temp_fp.easy, err); goto error_remove; } while (! temp_fp.paused && ! temp_fp.finished) if (wait_perform(&temp_fp) < 0) { save_errno = errno; goto error_remove; } if (temp_fp.finished && temp_fp.final_result != CURLE_OK) { save_errno = easy_errno(temp_fp.easy, temp_fp.final_result); goto error_remove; } // We've got a good response, close the original connection and // replace it with the new one. errm = curl_multi_remove_handle(fp->multi, fp->easy); if (errm != CURLM_OK) { // Clean up as much as possible curl_easy_reset(temp_fp.easy); if (curl_multi_remove_handle(fp->multi, temp_fp.easy) == CURLM_OK) { fp->nrunning--; curl_easy_cleanup(temp_fp.easy); } save_errno = multi_errno(errm); goto early_error; } fp->nrunning--; curl_easy_cleanup(fp->easy); fp->easy = temp_fp.easy; err = curl_easy_setopt(fp->easy, CURLOPT_WRITEDATA, fp); err |= curl_easy_setopt(fp->easy, CURLOPT_PRIVATE, fp); if (err != CURLE_OK) { save_errno = easy_errno(fp->easy, err); curl_easy_reset(fp->easy); errno = save_errno; return -1; } fp->buffer.len = 0; fp->paused = temp_fp.paused; fp->finished = temp_fp.finished; fp->perform_again = temp_fp.perform_again; fp->final_result = temp_fp.final_result; return 0; error_remove: curl_easy_reset(temp_fp.easy); // Ensure no pointers to on-stack temp_fp errm = curl_multi_remove_handle(fp->multi, temp_fp.easy); if (errm != CURLM_OK) { errno = multi_errno(errm); return -1; } fp->nrunning--; error: curl_easy_cleanup(temp_fp.easy); early_error: fp->can_seek = 0; // Don't try to seek again if (save_errno) errno = save_errno; return -1; } static int libcurl_close(hFILE *fpv) { hFILE_libcurl *fp = (hFILE_libcurl *) fpv; CURLcode err; CURLMcode errm; int save_errno = 0; // Before closing the file, unpause it and perform on it so that uploads // have the opportunity to signal EOF to the server -- see send_callback(). fp->buffer.len = 0; fp->closing = 1; fp->paused = 0; err = curl_easy_pause(fp->easy, CURLPAUSE_CONT); if (err != CURLE_OK) save_errno = easy_errno(fp->easy, err); while (save_errno == 0 && ! fp->paused && ! fp->finished) if (wait_perform(fp) < 0) save_errno = errno; if (fp->finished && fp->final_result != CURLE_OK) save_errno = easy_errno(fp->easy, fp->final_result); errm = curl_multi_remove_handle(fp->multi, fp->easy); if (errm != CURLM_OK && save_errno == 0) save_errno = multi_errno(errm); fp->nrunning--; curl_easy_cleanup(fp->easy); curl_multi_cleanup(fp->multi); if (fp->headers.callback) // Tell callback to free any data it needs to fp->headers.callback(fp->headers.callback_data, NULL); free_headers(&fp->headers.fixed, 1); free_headers(&fp->headers.extra, 1); if (save_errno) { errno = save_errno; return -1; } else return 0; } static const struct hFILE_backend libcurl_backend = { libcurl_read, libcurl_write, libcurl_seek, NULL, libcurl_close }; static hFILE * libcurl_open(const char *url, const char *modes, http_headers *headers) { hFILE_libcurl *fp; struct curl_slist *list; char mode; const char *s; CURLcode err; CURLMcode errm; int save, is_recursive; is_recursive = strchr(modes, 'R') != NULL; if ((s = strpbrk(modes, "rwa+")) != NULL) { mode = *s; if (strpbrk(&s[1], "rwa+")) mode = 'e'; } else mode = '\0'; if (mode != 'r' && mode != 'w') { errno = EINVAL; goto early_error; } fp = (hFILE_libcurl *) hfile_init(sizeof (hFILE_libcurl), modes, 0); if (fp == NULL) goto early_error; if (headers) { fp->headers = *headers; } else { memset(&fp->headers, 0, sizeof(fp->headers)); } fp->file_size = -1; fp->buffer.ptr.rd = NULL; fp->buffer.len = 0; fp->final_result = (CURLcode) -1; fp->paused = fp->closing = fp->finished = fp->perform_again = 0; fp->can_seek = 1; fp->tried_seek = 0; fp->delayed_seek = fp->last_offset = -1; fp->is_recursive = is_recursive; fp->nrunning = 0; fp->easy = NULL; fp->multi = curl_multi_init(); if (fp->multi == NULL) { errno = ENOMEM; goto error; } fp->easy = curl_easy_init(); if (fp->easy == NULL) { errno = ENOMEM; goto error; } // Make a route to the hFILE_libcurl* given just a CURL* easy handle err = curl_easy_setopt(fp->easy, CURLOPT_PRIVATE, fp); // Avoid many repeated CWD calls with FTP, instead requesting the filename // by full path (as done in knet, but not strictly compliant with RFC1738). err |= curl_easy_setopt(fp->easy, CURLOPT_FTP_FILEMETHOD, CURLFTPMETHOD_NOCWD); if (mode == 'r') { err |= curl_easy_setopt(fp->easy, CURLOPT_WRITEFUNCTION, recv_callback); err |= curl_easy_setopt(fp->easy, CURLOPT_WRITEDATA, fp); fp->is_read = 1; } else { err |= curl_easy_setopt(fp->easy, CURLOPT_READFUNCTION, send_callback); err |= curl_easy_setopt(fp->easy, CURLOPT_READDATA, fp); err |= curl_easy_setopt(fp->easy, CURLOPT_UPLOAD, 1L); if (append_header(&fp->headers.fixed, "Transfer-Encoding: chunked", 1) < 0) goto error; fp->is_read = 0; } err |= curl_easy_setopt(fp->easy, CURLOPT_SHARE, curl.share); err |= curl_easy_setopt(fp->easy, CURLOPT_URL, url); { char* env_curl_ca_bundle = getenv("CURL_CA_BUNDLE"); if (env_curl_ca_bundle) { err |= curl_easy_setopt(fp->easy, CURLOPT_CAINFO, env_curl_ca_bundle); } } err |= curl_easy_setopt(fp->easy, CURLOPT_USERAGENT, curl.useragent.s); if (fp->headers.callback) { if (add_callback_headers(fp) != 0) goto error; } if (get_auth_token(fp, url) < 0) goto error; if ((list = get_header_list(fp)) != NULL) err |= curl_easy_setopt(fp->easy, CURLOPT_HTTPHEADER, list); err |= curl_easy_setopt(fp->easy, CURLOPT_FOLLOWLOCATION, 1L); if (hts_verbose <= 8) err |= curl_easy_setopt(fp->easy, CURLOPT_FAILONERROR, 1L); if (hts_verbose >= 8) err |= curl_easy_setopt(fp->easy, CURLOPT_VERBOSE, 1L); if (err != 0) { errno = ENOSYS; goto error; } errm = curl_multi_add_handle(fp->multi, fp->easy); if (errm != CURLM_OK) { errno = multi_errno(errm); goto error; } fp->nrunning++; while (! fp->paused && ! fp->finished) if (wait_perform(fp) < 0) goto error_remove; if (fp->finished && fp->final_result != CURLE_OK) { errno = easy_errno(fp->easy, fp->final_result); goto error_remove; } if (mode == 'r') { double dval; if (curl_easy_getinfo(fp->easy, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dval) == CURLE_OK && dval >= 0.0) fp->file_size = (off_t) (dval + 0.1); } fp->base.backend = &libcurl_backend; return &fp->base; error_remove: save = errno; (void) curl_multi_remove_handle(fp->multi, fp->easy); fp->nrunning--; errno = save; error: save = errno; if (fp->easy) curl_easy_cleanup(fp->easy); if (fp->multi) curl_multi_cleanup(fp->multi); free_headers(&fp->headers.extra, 1); hfile_destroy((hFILE *) fp); errno = save; return NULL; early_error: return NULL; } static hFILE *hopen_libcurl(const char *url, const char *modes) { return libcurl_open(url, modes, NULL); } static int parse_va_list(http_headers *headers, va_list args) { const char *argtype; while ((argtype = va_arg(args, const char *)) != NULL) if (strcmp(argtype, "httphdr:v") == 0) { const char **hdr; for (hdr = va_arg(args, const char **); *hdr; hdr++) { if (append_header(&headers->fixed, *hdr, 1) < 0) return -1; if (is_authorization(*hdr)) headers->auth_hdr_num = -1; } } else if (strcmp(argtype, "httphdr:l") == 0) { const char *hdr; while ((hdr = va_arg(args, const char *)) != NULL) { if (append_header(&headers->fixed, hdr, 1) < 0) return -1; if (is_authorization(hdr)) headers->auth_hdr_num = -1; } } else if (strcmp(argtype, "httphdr") == 0) { const char *hdr = va_arg(args, const char *); if (hdr) { if (append_header(&headers->fixed, hdr, 1) < 0) return -1; if (is_authorization(hdr)) headers->auth_hdr_num = -1; } } else if (strcmp(argtype, "httphdr_callback") == 0) { headers->callback = va_arg(args, const hts_httphdr_callback); } else if (strcmp(argtype, "httphdr_callback_data") == 0) { headers->callback_data = va_arg(args, void *); } else if (strcmp(argtype, "va_list") == 0) { va_list *args2 = va_arg(args, va_list *); if (args2) { if (parse_va_list(headers, *args2) < 0) return -1; } } else if (strcmp(argtype, "auth_token_enabled") == 0) { const char *flag = va_arg(args, const char *); if (strcmp(flag, "false") == 0) headers->auth_hdr_num = -3; } else { errno = EINVAL; return -1; } return 0; } /* HTTP headers to be added to the request can be passed in as extra arguments to hopen(). The headers can be specified as follows: * Single header: hopen(url, mode, "httphdr", "X-Hdr-1: text", NULL); * Multiple headers in the argument list: hopen(url, mode, "httphdr:l", "X-Hdr-1: text", "X-Hdr-2: text", NULL, NULL); * Multiple headers in a char* array: hopen(url, mode, "httphdr:v", hdrs, NULL); where `hdrs` is a char **. The list ends with a NULL pointer. * A callback function hopen(url, mode, "httphdr_callback", func, "httphdr_callback_data", arg, NULL); `func` has type int (* hts_httphdr_callback) (void *cb_data, char ***hdrs); `arg` is passed to the callback as a void *. The function is called at file open, and when attempting to seek (which opens a new HTTP request). This allows, for example, access tokens that may have gone stale to be regenerated. The function is also called (with `hdrs` == NULL) on file close so that the callback can free any memory that it needs to. The callback should return 0 on success, non-zero on failure. It should return in *hdrs a list of strings containing the new headers (terminated with a NULL pointer). These will replace any headers previously supplied by the callback. If no changes are necessary, it can return NULL in *hdrs, in which case the previous headers will be left unchanged. Ownership of the strings in the header list passes to hfile_libcurl, so the callback should not attempt to use or free them itself. The memory containing the array belongs to the callback and will not be freed by hfile_libcurl. Headers supplied by the callback are appended after any specified using the "httphdr", "httphdr:l" or "httphdr:v" methods. No attempt is made to replace these headers (even if a key is repeated) so anything that is expected to vary needs to come from the callback. */ static hFILE *vhopen_libcurl(const char *url, const char *modes, va_list args) { hFILE *fp = NULL; http_headers headers = { { NULL, 0, 0 }, { NULL, 0, 0 }, NULL, NULL }; if (parse_va_list(&headers, args) == 0) { fp = libcurl_open(url, modes, &headers); } if (!fp) { free_headers(&headers.fixed, 1); } return fp; } int PLUGIN_GLOBAL(hfile_plugin_init,_libcurl)(struct hFILE_plugin *self) { static const struct hFILE_scheme_handler handler = { hopen_libcurl, hfile_always_remote, "libcurl", 2000 + 50, vhopen_libcurl }; #ifdef ENABLE_PLUGINS // Embed version string for examination via strings(1) or what(1) static const char id[] = "@(#)hfile_libcurl plugin (htslib)\t" HTS_VERSION; const char *version = strchr(id, '\t')+1; #else const char *version = hts_version(); #endif const curl_version_info_data *info; const char * const *protocol; const char *auth; CURLcode err; CURLSHcode errsh; err = curl_global_init(CURL_GLOBAL_ALL); if (err != CURLE_OK) { errno = easy_errno(NULL, err); return -1; } curl.share = curl_share_init(); if (curl.share == NULL) { curl_global_cleanup(); errno = EIO; return -1; } errsh = curl_share_setopt(curl.share, CURLSHOPT_LOCKFUNC, share_lock); errsh |= curl_share_setopt(curl.share, CURLSHOPT_UNLOCKFUNC, share_unlock); errsh |= curl_share_setopt(curl.share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); if (errsh != 0) { curl_share_cleanup(curl.share); curl_global_cleanup(); errno = EIO; return -1; } if ((auth = getenv("HTS_AUTH_LOCATION")) != NULL) { curl.auth_path = strdup(auth); curl.auth_map = kh_init(auth_map); if (!curl.auth_path || !curl.auth_map) { int save_errno = errno; free(curl.auth_path); kh_destroy(auth_map, curl.auth_map); curl_share_cleanup(curl.share); curl_global_cleanup(); errno = save_errno; return -1; } } if ((auth = getenv("HTS_ALLOW_UNENCRYPTED_AUTHORIZATION_HEADER")) != NULL && strcmp(auth, "I understand the risks") == 0) { curl.allow_unencrypted_auth_header = 1; } info = curl_version_info(CURLVERSION_NOW); ksprintf(&curl.useragent, "htslib/%s libcurl/%s", version, info->version); self->name = "libcurl"; self->destroy = libcurl_exit; for (protocol = info->protocols; *protocol; protocol++) hfile_add_scheme_handler(*protocol, &handler); return 0; }