Interface for downloading HTTP, FTP documents. More...

#include "cpl_conv.h"
#include "cpl_string.h"
#include "cpl_progress.h"
#include "cpl_vsi.h"

Classes
struct	CPLMimePart

struct	CPLHTTPResult

class	GOA2Manager
	Manager of Google OAuth2 authentication. More...

Functions
int	CPLHTTPEnabled (void)
	Return if CPLHTTP services can be useful. More...

CPLHTTPResult *	CPLHTTPFetch (const char *pszURL, CSLConstList papszOptions)
	Fetch a document from an url and return in a string. More...

CPLHTTPResult *	CPLHTTPFetchEx (const char pszURL, CSLConstList papszOptions, GDALProgressFunc pfnProgress, void pProgressArg, CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg)
	Fetch a document from an url and return in a string. More...

CPLHTTPResult **	CPLHTTPMultiFetch (const char const papszURL, int nURLCount, int nMaxSimultaneous, CSLConstList papszOptions)
	Fetch several documents at once. More...

void	CPLHTTPCleanup (void)
	Cleanup function to call at application termination.

void	CPLHTTPDestroyResult (CPLHTTPResult *psResult)
	Clean the memory associated with the return value of CPLHTTPFetch() More...

void	CPLHTTPDestroyMultiResult (CPLHTTPResult **papsResults, int nCount)
	Clean the memory associated with the return value of CPLHTTPMultiFetch() More...

int	CPLHTTPParseMultipartMime (CPLHTTPResult *psResult)
	Parses a MIME multipart message. More...

char *	GOA2GetAuthorizationURL (const char *pszScope)
	Return authorization url for a given scope. More...

char *	GOA2GetRefreshToken (const char pszAuthToken, const char pszScope)
	Turn Auth Token into a Refresh Token. More...

char *	GOA2GetAccessToken (const char pszRefreshToken, const char pszScope)
	Fetch access token using refresh token. More...

char **	GOA2GetAccessTokenFromServiceAccount (const char pszPrivateKey, const char pszClientEmail, const char *pszScope, CSLConstList papszAdditionalClaims, CSLConstList papszOptions)
	Fetch access token using Service Account OAuth2. More...

char **	GOA2GetAccessTokenFromCloudEngineVM (CSLConstList papszOptions)
	Fetch access token using Cloud Engine internal REST API. More...

bool	CPLIsMachinePotentiallyGCEInstance ()
	Returns whether the current machine is potentially a Google Compute Engine instance. More...

bool	CPLIsMachineForSureGCEInstance ()
	Returns whether the current machine is surely a Google Compute Engine instance. More...

Detailed Description

Interface for downloading HTTP, FTP documents.

Function Documentation

◆ CPLHTTPDestroyMultiResult()

void CPLHTTPDestroyMultiResult	(	CPLHTTPResult **	papsResults,
		int	nCount
	)

Clean the memory associated with the return value of CPLHTTPMultiFetch()

Parameters

papsResults	pointer to the return value of CPLHTTPMultiFetch()
nCount	value of the nURLCount parameter passed to CPLHTTPMultiFetch()

Since: GDAL 2.3

◆ CPLHTTPDestroyResult()

void CPLHTTPDestroyResult ( CPLHTTPResult * psResult )

Clean the memory associated with the return value of CPLHTTPFetch()

Parameters

psResult pointer to the return value of CPLHTTPFetch()

◆ CPLHTTPEnabled()

int CPLHTTPEnabled ( void )

Return if CPLHTTP services can be useful.

Those services depend on GDAL being build with libcurl support.

Returns: TRUE if libcurl support is enabled

◆ CPLHTTPFetch()

CPLHTTPResult* CPLHTTPFetch	(	const char *	pszURL,
		CSLConstList	papszOptions
	)

Fetch a document from an url and return in a string.

Parameters

pszURL valid URL recognized by underlying download library (libcurl)

papszOptions

option list as a NULL-terminated array of strings. May be NULL. The following options are handled :

CONNECTTIMEOUT=val, where val is in seconds (possibly with decimals). This is the maximum delay for the connection to be established before being aborted (GDAL >= 2.2).
TIMEOUT=val, where val is in seconds. This is the maximum delay for the whole request to complete before being aborted.
LOW_SPEED_TIME=val, where val is in seconds. This is the maximum time where the transfer speed should be below the LOW_SPEED_LIMIT (if not specified 1b/s), before the transfer to be considered too slow and aborted. (GDAL >= 2.1)
LOW_SPEED_LIMIT=val, where val is in bytes/second. See LOW_SPEED_TIME. Has only effect if LOW_SPEED_TIME is specified too. (GDAL >= 2.1)
HEADERS=val, where val is an extra header to use when getting a web page. For example "Accept: application/x-ogcwkt"
HEADER_FILE=filename: filename of a text file with "key: value" headers. (GDAL >= 2.2)
HTTPAUTH=[BASIC/NTLM/GSSNEGOTIATE/ANY] to specify an authentication scheme to use.
USERPWD=userid:password to specify a user and password for authentication
POSTFIELDS=val, where val is a nul-terminated string to be passed to the server with a POST request.
PROXY=val, to make requests go through a proxy server, where val is of the form proxy.server.com:port_number. This option affects both HTTP and HTTPS URLs.
HTTPS_PROXY=val (GDAL >= 2.4), the same meaning as PROXY, but this option is taken into account only for HTTPS URLs.
PROXYUSERPWD=val, where val is of the form username:password
PROXYAUTH=[BASIC/NTLM/DIGEST/ANY] to specify an proxy authentication scheme to use.
NETRC=[YES/NO] to enable or disable use of $HOME/.netrc, default YES.
CUSTOMREQUEST=val, where val is GET, PUT, POST, DELETE, etc.. (GDAL >= 1.9.0)
COOKIE=val, where val is formatted as COOKIE1=VALUE1; COOKIE2=VALUE2; ...
COOKIEFILE=val, where val is file name to read cookies from (GDAL >= 2.4)
COOKIEJAR=val, where val is file name to store cookies to (GDAL >= 2.4)
MAX_RETRY=val, where val is the maximum number of retry attempts if a 429, 502, 503 or 504 HTTP error occurs. Default is 0. (GDAL >= 2.0)
RETRY_DELAY=val, where val is the number of seconds between retry attempts. Default is 30. (GDAL >= 2.0)
MAX_FILE_SIZE=val, where val is a number of bytes (GDAL >= 2.2)
CAINFO=/path/to/bundle.crt. This is path to Certificate Authority (CA) bundle file. By default, it will be looked in a system location. If the CAINFO options is not defined, GDAL will also look if the CURL_CA_BUNDLE environment variable is defined to use it as the CAINFO value, and as a fallback to the SSL_CERT_FILE environment variable. (GDAL >= 2.1.3)
HTTP_VERSION=1.0/1.1/2/2TLS (GDAL >= 2.3). Specify HTTP version to use. Will default to 1.1 generally (except on some controlled environments, like Google Compute Engine VMs, where 2TLS will be the default). Support for HTTP/2 requires curl 7.33 or later, built against nghttp2. "2TLS" means that HTTP/2 will be attempted for HTTPS connections only. Whereas "2" means that HTTP/2 will be attempted for HTTP or HTTPS.
SSL_VERIFYSTATUS=YES/NO (GDAL >= 2.3, and curl >= 7.41): determines whether the status of the server cert using the "Certificate Status Request" TLS extension (aka. OCSP stapling) should be checked. If this option is enabled but the server does not support the TLS extension, the verification will fail. Default to NO.
USE_CAPI_STORE=YES/NO (GDAL >= 2.3, Windows only): whether CA certificates from the Windows certificate store. Defaults to NO.

Alternatively, if not defined in the papszOptions arguments, the CONNECTTIMEOUT, TIMEOUT, LOW_SPEED_TIME, LOW_SPEED_LIMIT, USERPWD, PROXY, HTTPS_PROXY, PROXYUSERPWD, PROXYAUTH, NETRC, MAX_RETRY and RETRY_DELAY, HEADER_FILE, HTTP_VERSION, SSL_VERIFYSTATUS, USE_CAPI_STORE values are searched in the configuration options respectively named GDAL_HTTP_CONNECTTIMEOUT, GDAL_HTTP_TIMEOUT, GDAL_HTTP_LOW_SPEED_TIME, GDAL_HTTP_LOW_SPEED_LIMIT, GDAL_HTTP_USERPWD, GDAL_HTTP_PROXY, GDAL_HTTPS_PROXY, GDAL_HTTP_PROXYUSERPWD, GDAL_PROXY_AUTH, GDAL_HTTP_NETRC, GDAL_HTTP_MAX_RETRY, GDAL_HTTP_RETRY_DELAY, GDAL_HTTP_HEADER_FILE, GDAL_HTTP_VERSION, GDAL_HTTP_SSL_VERIFYSTATUS, GDAL_HTTP_USE_CAPI_STORE

Returns: a CPLHTTPResult* structure that must be freed by CPLHTTPDestroyResult(), or NULL if libcurl support is disabled

◆ CPLHTTPFetchEx()

CPLHTTPResult* CPLHTTPFetchEx	(	const char *	pszURL,
		CSLConstList	papszOptions,
		GDALProgressFunc	pfnProgress,
		void *	pProgressArg,
		CPLHTTPFetchWriteFunc	pfnWrite,
		void *	pWriteArg
	)

Fetch a document from an url and return in a string.

Parameters

pszURL	Url to fetch document from web.
papszOptions	Option list as a NULL-terminated array of strings. Available keys see in CPLHTTPFetch.
pfnProgress	Callback for reporting algorithm progress matching the GDALProgressFunc() semantics. May be NULL.
pProgressArg	Callback argument passed to pfnProgress.
pfnWrite	Write function pointer matching the CPLHTTPWriteFunc() semantics. May be NULL.
pWriteArg	Argument which will pass to a write function.

Returns: A CPLHTTPResult* structure that must be freed by CPLHTTPDestroyResult(), or NULL if libcurl support is disabled.

◆ CPLHTTPMultiFetch()

CPLHTTPResult** CPLHTTPMultiFetch	(	const char const	papszURL,
		int	nURLCount,
		int	nMaxSimultaneous,
		CSLConstList	papszOptions
	)

Fetch several documents at once.

Parameters

papszURL	array of valid URLs recognized by underlying download library (libcurl)
nURLCount	number of URLs of papszURL
nMaxSimultaneous	maximum number of downloads to issue simultaneously. Any negative or zer value means unlimited.
papszOptions	option list as a NULL-terminated array of strings. May be NULL. Refer to CPLHTTPFetch() for valid options.

Returns: an array of CPLHTTPResult* structures that must be freed by CPLHTTPDestroyMultiResult() or NULL if libcurl support is disabled

Since: GDAL 2.3

◆ CPLHTTPParseMultipartMime()

int CPLHTTPParseMultipartMime ( CPLHTTPResult * psResult )

Parses a MIME multipart message.

This function will iterate over each part and put it in a separate element of the pasMimePart array of the provided psResult structure.

Parameters

psResult pointer to the return value of CPLHTTPFetch()

Returns: TRUE if the message contains MIME multipart message.

◆ CPLIsMachineForSureGCEInstance()

bool CPLIsMachineForSureGCEInstance ( )

Returns whether the current machine is surely a Google Compute Engine instance.

This does a very quick check without network access. Note: only works for Linux GCE instances.

Returns: true if the current machine is surely a GCE instance.

Since: GDAL 2.3

◆ CPLIsMachinePotentiallyGCEInstance()

bool CPLIsMachinePotentiallyGCEInstance ( )

Returns whether the current machine is potentially a Google Compute Engine instance.

This does a very quick check without network access. To confirm if the machine is effectively a GCE instance, metadata.google.internal must be queried.

Returns: true if the current machine is potentially a GCE instance.

Since: GDAL 2.3

◆ GOA2GetAccessToken()

char* GOA2GetAccessToken	(	const char *	pszRefreshToken,
		const char *	pszScope
	)

Fetch access token using refresh token.

The permanent refresh token is used to fetch a temporary (usually one hour) access token using Google OAuth2 web services.

A CPLError will be reported if the request fails for some reason. Common reasons include the refresh token having been revoked by the user or http connection problems.

Parameters

pszRefreshToken	the refresh token from GOA2GetRefreshToken().
pszScope	the scope for which it is valid. Currently unused

Returns: access token, to be freed with CPLFree(), null on failure.

◆ GOA2GetAccessTokenFromCloudEngineVM()

char** GOA2GetAccessTokenFromCloudEngineVM ( CSLConstList papszOptions )

Fetch access token using Cloud Engine internal REST API.

The default service accounts bound to the current Google Cloud Engine VM is used for OAuth2 authentication

A CPLError will be reported if the request fails for some reason. Common reasons include the refresh token having been revoked by the user or http connection problems.

Parameters

papszOptions NULL terminated list of options. None currently

Returns: a list of key=value pairs, including a access_token and expires_in

Since: GDAL 2.3

◆ GOA2GetAccessTokenFromServiceAccount()

char** GOA2GetAccessTokenFromServiceAccount	(	const char *	pszPrivateKey,
		const char *	pszClientEmail,
		const char *	pszScope,
		CSLConstList	papszAdditionalClaims,
		CSLConstList	papszOptions
	)

Fetch access token using Service Account OAuth2.

See https://developers.google.com/identity/protocols/OAuth2ServiceAccount

A CPLError will be reported if the request fails for some reason.

Parameters

pszPrivateKey	Private key as a RSA private key
pszClientEmail	Client email
pszScope	the service being requested
papszAdditionalClaims	additional claims, or NULL
papszOptions	NULL terminated list of options. None currently

Returns: a list of key=value pairs, including a access_token and expires_in

Since: GDAL 2.3

See https://developers.google.com/identity/protocols/OAuth2ServiceAccount and https://jwt.io/

◆ GOA2GetAuthorizationURL()

char* GOA2GetAuthorizationURL ( const char * pszScope )

Return authorization url for a given scope.

Returns the URL that a user should visit, and use for authentication in order to get an "auth token" indicating their willingness to use a service.

Note that when the user visits this url they will be asked to login (using a google/gmail/etc) account, and to authorize use of the requested scope for the application "GDAL/OGR". Once they have done so, they will be presented with a lengthy string they should "enter into their application". This is the "auth token" to be passed to GOA2GetRefreshToken(). The "auth token" can only be used once.

This function should never fail.

Parameters

pszScope the service being requested, not yet URL encoded, such as "https://www.googleapis.com/auth/fusiontables".

Returns: the URL to visit - should be freed with CPLFree().

◆ GOA2GetRefreshToken()

char* GOA2GetRefreshToken	(	const char *	pszAuthToken,
		const char *	pszScope
	)

Turn Auth Token into a Refresh Token.

A one time "auth token" provided by the user is turned into a reusable "refresh token" using a google oauth2 web service.

A CPLError will be reported if the translation fails for some reason. Common reasons include the auth token already having been used before, it not being appropriate for the passed scope and configured client api or http connection problems. NULL is returned on error.

Parameters

pszAuthToken	the authorization token from the user.
pszScope	the scope for which it is valid.

Returns: refresh token, to be freed with CPLFree(), null on failure.

Classes

Functions

Detailed Description

Function Documentation

◆ CPLHTTPDestroyMultiResult()

◆ CPLHTTPDestroyResult()

◆ CPLHTTPEnabled()

◆ CPLHTTPFetch()

◆ CPLHTTPFetchEx()

◆ CPLHTTPMultiFetch()

◆ CPLHTTPParseMultipartMime()

◆ CPLIsMachineForSureGCEInstance()

◆ CPLIsMachinePotentiallyGCEInstance()

◆ GOA2GetAccessToken()

◆ GOA2GetAccessTokenFromCloudEngineVM()

◆ GOA2GetAccessTokenFromServiceAccount()

◆ GOA2GetAuthorizationURL()

◆ GOA2GetRefreshToken()