jli  Linuxx86_641.10.3v1.10.30b4590a5507d3f3046e5bafc007cacbbfc9b310bHTTPP5>U638Z*/opt/julia/packages/HTTP/sJD5V/src/HTTP.jlٗA_䇽UlD*Base64ސݗ1V$ bdSocketsj2 EY8pDatesԱ6;V.OG'\URIs6l7vSzԗLoggingExtrasmspAQ)sMbedTLS8tCI1MOpenSSL0/opt/julia/packages/HTTP/sJD5V/src/Conditions.jlٗA0/opt/julia/packages/HTTP/sJD5V/src/access_log.jlbA+/opt/julia/packages/HTTP/sJD5V/src/Pairs.jlٗA./opt/julia/packages/HTTP/sJD5V/src/IOExtras.jlٗAސݗ1V$ bdSocketsIOExtrasmspAQ)sMbedTLSIOExtras8tCI1MOpenSSLIOExtras-/opt/julia/packages/HTTP/sJD5V/src/Strings.jlٗA0/opt/julia/packages/HTTP/sJD5V/src/Exceptions.jlٗA6l7vSzԗLoggingExtras Exceptions$#DYQJkConcurrentUtilities Connections1/opt/julia/packages/HTTP/sJD5V/src/StatusCodes.jlٗA5/opt/julia/packages/HTTP/sJD5V/src/status_messages.jlbA StatusCodes./opt/julia/packages/HTTP/sJD5V/src/Messages.jlٗAԱ6;V.OG'\URIsMessages1SZ\xfKCodecZlibMessages-/opt/julia/packages/HTTP/sJD5V/src/cookies.jlbAj2 EY8pDatesCookiesԱ6;V.OG'\URIsCookies6l7vSzԗLoggingExtrasCookiesސݗ1V$ bdSocketsCookies//opt/julia/packages/HTTP/sJD5V/src/cookiejar.jlbACookies-/opt/julia/packages/HTTP/sJD5V/src/Streams.jlٗAސݗ1V$ bdSocketsStreams6l7vSzԗLoggingExtrasStreamsA/opt/julia/packages/HTTP/sJD5V/src/clientlayers/MessageRequest.jlbAԱ6;V.OG'\URIsMessageRequest6l7vSzԗLoggingExtrasMessageRequestB/opt/julia/packages/HTTP/sJD5V/src/clientlayers/RedirectRequest.jlbAԱ6;V.OG'\URIsRedirectRequest6l7vSzԗLoggingExtrasRedirectRequestA/opt/julia/packages/HTTP/sJD5V/src/clientlayers/HeadersRequest.jlbA_䇽UlD*Base64HeadersRequestԱ6;V.OG'\URIsHeadersRequest6l7vSzԗLoggingExtrasHeadersRequest@/opt/julia/packages/HTTP/sJD5V/src/clientlayers/CookieRequest.jlbAj2 EY8pDates CookieRequest6l7vSzԗLoggingExtras CookieRequestԱ6;V.OG'\URIs CookieRequestA/opt/julia/packages/HTTP/sJD5V/src/clientlayers/TimeoutRequest.jlbA6l7vSzԗLoggingExtrasTimeoutRequest$R>#DYQJkConcurrentUtilitiesTimeoutRequestC/opt/julia/packages/HTTP/sJD5V/src/clientlayers/ExceptionRequest.jlbA?/opt/julia/packages/HTTP/sJD5V/src/clientlayers/RetryRequest.jlbAސݗ1V$ bdSockets RetryRequest6l7vSzԗLoggingExtras RetryRequestmspAQ)sMbedTLS RetryRequest8tCI1MOpenSSL RetryRequest$#DYQJkConcurrentUtilitiesConnectionRequest$#DYQJkConcurrentUtilities StreamRequest./opt/julia/packages/HTTP/sJD5V/src/download.jlbA1SZ\xfKCodecZlib-/opt/julia/packages/HTTP/sJD5V/src/Servers.jlٗAސݗ1V$ bdSocketsServershUXM=T{VLoggingServers6l7vSzԗLoggingExtrasServersmspAQ)sMbedTLSServersj2 EY8pDatesServers./opt/julia/packages/HTTP/sJD5V/src/Handlers.jlٗAԱ6;V.OG'\URIsHandlers4/opt/julia/packages/HTTP/sJD5V/src/parsemultipart.jlbA0/opt/julia/packages/HTTP/sJD5V/src/WebSockets.jlٗA_䇽UlD*Base64 WebSockets6l7vSzԗLoggingExtras WebSocketsrz9[viqUUIDs WebSocketsސݗ1V$ bdSockets WebSockets\Y_ɢ?Random WebSocketsmspAQ)sMbedTLS WebSocketsE$CoremуJ5Basemу]J5MainmуJ5ArgToolsBń x(mуF K5 Artifactsmr-V3|mу K5Base64UlD*_mу> K5CRC32c\y.jmуj K5 FileWatchingXzsy`{,zmуh& K5LibdluVW59˗,mу-" K5LoggingT{VhUXM=mуrU" K5MmapP~:xg,Omу|' K5NetworkOptionsC0YW,mуʠ, K5SHAQ<$!<%mу1 K5 Serialization [)*k1mу-G K5Sockets1V$ bdސݗmуYBY K5UnicodeP>I>Nrmуeszo K5 LinearAlgebraSm7̏mуuux K5 OpenBLAS_jll[(Śb6EcQ FmуDux K5libblastrampoline_jllLSۆ }lxӠmу^} K5MarkdownZPn7z`smу/Ed~ K5Printfg^cX׸QDmу;h K5Random_ɢ?\Ymу? K5TarOi>աmу!t, K5DatesEY8pj2 mуX K5FuturebS;3{I xVMmуsD K5InteractiveUtilsWL ~@'ZmуVg K5LibGit2Z[&RPTv3EКRmу8J K5 LibGit2_jll YXg}]$mуD K5 MbedTLS_jllAX 3ȡ_mу- K5 LibSSH2_jlloTZk)߆#V"\"6TranscodingStreams(P;BŦ&߸Y07TestExtRm()Dy~W 97Zlib_jll?QXZwzTחNUO5 CodecZlibZ\xfK1Sj=8SimpleBufferStreamKTzw8P"\(-6s v) _@: Aa ! 0generic*/opt/julia/packages/HTTP/sJD5V/src/HTTP.jlimodule HTTP export startwrite, startread, closewrite, closeread, @logfmt_str, common_logfmt, combined_logfmt, WebSockets const DEBUG_LEVEL = Ref(0) Base.@deprecate escape escapeuri using Base64, Sockets, Dates, URIs, LoggingExtras, MbedTLS, OpenSSL function access_threaded(f, v::Vector) tid = Threads.threadid() 0 < tid <= length(v) || _length_assert() if @inbounds isassigned(v, tid) @inbounds x = v[tid] else x = f() @inbounds v[tid] = x end return x end @noinline _length_assert() = @assert false "0 < tid <= v" function open end const SOCKET_TYPE_TLS = Ref{Any}(OpenSSL.SSLStream) include("Conditions.jl") ;using .Conditions include("access_log.jl") include("Pairs.jl") ;using .Pairs include("IOExtras.jl") ;using .IOExtras include("Strings.jl") ;using .Strings include("Exceptions.jl") ;using .Exceptions include("sniff.jl") ;using .Sniff include("multipart.jl") ;using .Forms include("Parsers.jl") ;import .Parsers: Headers, Header, ParseError include("Connections.jl") ;using .Connections # backwards compat const ConnectionPool = Connections include("StatusCodes.jl") ;using .StatusCodes include("Messages.jl") ;using .Messages include("cookies.jl") ;using .Cookies include("Streams.jl") ;using .Streams getrequest(r::Request) = r getrequest(s::Stream) = s.message.request # Wraps client-side "layer" to track the amount of time spent in it as a request is processed. function observelayer(f) function observation(req_or_stream; kw...) req = getrequest(req_or_stream) nm = nameof(f) cntnm = Symbol(nm, "_count") durnm = Symbol(nm, "_duration_ms") start_time = time() req.context[cntnm] = Base.get(req.context, cntnm, 0) + 1 try return f(req_or_stream; kw...) finally req.context[durnm] = Base.get(req.context, durnm, 0) + (time() - start_time) * 1000 # @info "observed layer = $f, count = $(req.context[cntnm]), duration = $(req.context[durnm])" end end end include("clientlayers/MessageRequest.jl"); using .MessageRequest include("clientlayers/RedirectRequest.jl"); using .RedirectRequest include("clientlayers/HeadersRequest.jl"); using .HeadersRequest include("clientlayers/CookieRequest.jl"); using .CookieRequest include("clientlayers/TimeoutRequest.jl"); using .TimeoutRequest include("clientlayers/ExceptionRequest.jl"); using .ExceptionRequest include("clientlayers/RetryRequest.jl"); using .RetryRequest include("clientlayers/ConnectionRequest.jl"); using .ConnectionRequest include("clientlayers/StreamRequest.jl"); using .StreamRequest include("download.jl") include("Servers.jl") ;using .Servers; using .Servers: listen include("Handlers.jl") ;using .Handlers; using .Handlers: serve include("parsemultipart.jl") ;using .MultiPartParsing: parse_multipart_form include("WebSockets.jl") ;using .WebSockets const nobody = UInt8[] """ HTTP.request(method, url [, headers [, body]]; ]) -> HTTP.Response Send a HTTP Request Message and receive a HTTP Response Message. e.g. ```julia r = HTTP.request("GET", "http://httpbin.org/ip") r = HTTP.get("http://httpbin.org/ip") # equivalent shortcut println(r.status) println(String(r.body)) ``` `headers` can be any collection where `[string(k) => string(v) for (k,v) in headers]` yields `Vector{Pair}`. e.g. a `Dict()`, a `Vector{Tuple}`, a `Vector{Pair}` or an iterator. By convention, if a header _value_ is an empty string, it will not be written when sending a request (following the curl convention). By default, a copy of provided headers is made (since required headers are typically set during the request); to avoid this copy and have HTTP.jl mutate the provided headers array, pass `copyheaders=false` as an additional keyword argument to the request. The `body` argument can be a variety of objects: - an `AbstractDict` or `NamedTuple` to be serialized as the "application/x-www-form-urlencoded" content type - any `AbstractString` or `AbstractVector{UInt8}` which will be sent "as is" for the request body - a readable `IO` stream or any `IO`-like type `T` for which `eof(T)` and `readavailable(T)` are defined. This stream will be read and sent until `eof` is `true`. This object should support the `mark`/`reset` methods if request retries are desired (if not, no retries will be attempted). - Any collection or iterable of the above (`AbstractDict`, `AbstractString`, `AbstractVector{UInt8}`, or `IO`) which will result in a "chunked" request body, where each iterated element will be sent as a separate chunk - a [`HTTP.Form`](@ref), which will be serialized as the "multipart/form-data" content-type The `HTTP.Response` struct contains: - `status::Int16` e.g. `200` - `headers::Vector{Pair{String,String}}` e.g. ["Server" => "Apache", "Content-Type" => "text/html"] - `body::Vector{UInt8}` or `::IO`, the Response Body bytes or the `io` argument provided via the `response_stream` keyword argument Functions `HTTP.get`, `HTTP.put`, `HTTP.post` and `HTTP.head` are defined as shorthand for `HTTP.request("GET", ...)`, etc. Supported optional keyword arguments: - `query = nothing`, a `Pair` or `Dict` of key => values to be included in the url - `response_stream = nothing`, a writeable `IO` stream or any `IO`-like type `T` for which `write(T, AbstractVector{UInt8})` is defined. The response body will be written to this stream instead of returned as a `Vector{UInt8}`. - `verbose = 0`, set to `1` or `2` for increasingly verbose logging of the request and response process - `connect_timeout = 30`, close the connection after this many seconds if it is still attempting to connect. Use `connect_timeout = 0` to disable. - `pool = nothing`, an `HTTP.Pool` object to use for managing the reuse of connections between requests. By default, a global pool is used, which is shared across all requests. To create a pool for a specific set of requests, use `pool = HTTP.Pool(max::Int)`, where `max` controls the maximum number of concurrent connections allowed to be used for requests at a given time. - `readtimeout = 0`, abort a request after this many seconds. Will trigger retries if applicable. Use `readtimeout = 0` to disable. - `status_exception = true`, throw `HTTP.StatusError` for response status >= 300. - Basic authentication is detected automatically from the provided url's `userinfo` (in the form `scheme://user:password@host`) and adds the `Authorization: Basic` header; this can be disabled by passing `basicauth=false` - `canonicalize_headers = false`, rewrite request and response headers in Canonical-Camel-Dash-Format. - `proxy = proxyurl`, pass request through a proxy given as a url; alternatively, the `http_proxy`, `HTTP_PROXY`, `https_proxy`, `HTTPS_PROXY`, and `no_proxy` environment variables are also detected/used; if set, they will be used automatically when making requests. - `detect_content_type = false`: if `true` and the request body is not a form or `IO`, it will be inspected and the "Content-Type" header will be set to the detected content type. - `decompress = nothing`, by default, decompress the response body if the response has a "Content-Encoding" header set to "gzip". If `decompress=true`, decompress the response body regardless of `Content-Encoding` header. If `decompress=false`, do not decompress the response body. - `logerrors = false`, if `true`, `HTTP.StatusError`, `HTTP.TimeoutError`, `HTTP.IOError`, and `HTTP.ConnectError` will be logged via `@error` as they happen, regardless of whether the request is then retried or not. Useful for debugging or monitoring requests where there's worry of certain errors happening but ignored because of retries. - `logtag = nothing`, if provided, will be used as the tag for error logging. Useful for debugging or monitoring requests. - `observelayers = false`, if `true`, enables the `HTTP.observelayer` to wrap each client-side "layer" to track the amount of time spent in each layer as a request is processed. This can be useful for debugging performance issues. Note that when retries or redirects happen, the time spent in each layer is cumulative, as noted by the `[layer]_count`. The metrics are stored in the `Request.context` dictionary, and can be accessed like `HTTP.get(...).request.context` Retry arguments: - `retry = true`, retry idempotent requests in case of error. - `retries = 4`, number of times to retry. - `retry_non_idempotent = false`, retry non-idempotent requests too. e.g. POST. - `retry_delays = ExponentialBackOff(n = retries)`, provide a custom `ExponentialBackOff` object to control the delay between retries. - `retry_check = (s, ex, req, resp, resp_body) -> Bool`, provide a custom function to control whether a retry should be attempted. The function should accept 5 arguments: the delay state, exception, request, response (an `HTTP.Response` object *if* a request was successfully made, otherwise `nothing`), and `resp_body` response body (which may be `nothing` if there is no response yet, otherwise a `Vector{UInt8}`), and return `true` if a retry should be attempted. Redirect arguments: - `redirect = true`, follow 3xx redirect responses; i.e. additional requests will be made to the redirected location - `redirect_limit = 3`, maximum number of times a redirect will be followed - `redirect_method = nothing`, the method to use for the redirected request; by default, GET will be used, only responses with 307/308 will use the same original request method. Pass `redirect_method=:same` to pass the same method as the orginal request though note that some servers may not respond/accept the same method. It's also valid to pass the exact method to use as a string, like `redirect_method="PUT"`. - `forwardheaders = true`, forward original headers on redirect. SSL arguments: - `require_ssl_verification = NetworkOptions.verify_host(host)`, pass `MBEDTLS_SSL_VERIFY_REQUIRED` to the mbed TLS library. ["... peer must present a valid certificate, handshake is aborted if verification failed."](https://tls.mbed.org/api/ssl_8h.html#a5695285c9dbfefec295012b566290f37) - `sslconfig = SSLConfig(require_ssl_verification)` - `socket_type_tls = MbedTLS.SSLContext`, the type of socket to use for TLS connections. Defaults to `MbedTLS.SSLContext`. Also supported is passing `socket_type_tls = OpenSSL.SSLStream`. To change the global default, set `HTTP.SOCKET_TYPE_TLS[] = OpenSSL.SSLStream`. Cookie arguments: - `cookies::Union{Bool, Dict{<:AbstractString, <:AbstractString}} = true`, enable cookies, or alternatively, pass a `Dict{AbstractString, AbstractString}` of name-value pairs to manually pass cookies in the request "Cookie" header - `cookiejar::HTTP.CookieJar=HTTP.COOKIEJAR`: threadsafe cookie jar struct for keeping track of cookies per host; a global cookie jar is used by default. ## Request Body Examples String body: ```julia HTTP.request("POST", "http://httpbin.org/post", [], "post body data") ``` Stream body from file: ```julia io = open("post_data.txt", "r") HTTP.request("POST", "http://httpbin.org/post", [], io) ``` Generator body: ```julia chunks = ("chunk\$i" for i in 1:1000) HTTP.request("POST", "http://httpbin.org/post", [], chunks) ``` Collection body: ```julia chunks = [preamble_chunk, data_chunk, checksum(data_chunk)] HTTP.request("POST", "http://httpbin.org/post", [], chunks) ``` `open() do io` body: ```julia HTTP.open("POST", "http://httpbin.org/post") do io write(io, preamble_chunk) write(io, data_chunk) write(io, checksum(data_chunk)) end ``` ## Response Body Examples String body: ```julia r = HTTP.request("GET", "http://httpbin.org/get") println(String(r.body)) ``` Stream body to file: ```julia io = open("get_data.txt", "w") r = HTTP.request("GET", "http://httpbin.org/get", response_stream=io) close(io) println(read("get_data.txt")) ``` Stream body through buffer: ```julia r = HTTP.get("http://httpbin.org/get", response_stream=IOBuffer()) println(String(take!(r.body))) ``` Stream body through `open() do io`: ```julia r = HTTP.open("GET", "http://httpbin.org/stream/10") do io while !eof(io) println(String(readavailable(io))) end end HTTP.open("GET", "https://tinyurl.com/bach-cello-suite-1-ogg") do http n = 0 r = startread(http) l = parse(Int, HTTP.header(r, "Content-Length")) open(`vlc -q --play-and-exit --intf dummy -`, "w") do vlc while !eof(http) bytes = readavailable(http) write(vlc, bytes) n += length(bytes) println("streamed \$n-bytes \$((100*n)÷l)%\\u1b[1A") end end end ``` Interfacing with RESTful JSON APIs: ```julia using JSON params = Dict("user"=>"RAO...tjN", "token"=>"NzU...Wnp", "message"=>"Hello!") url = "http://api.domain.com/1/messages.json" r = HTTP.post(url, body=JSON.json(params)) println(JSON.parse(String(r.body))) ``` Stream bodies from and to files: ```julia in = open("foo.png", "r") out = open("foo.jpg", "w") HTTP.request("POST", "http://convert.com/png2jpg", [], in, response_stream=out) ``` Stream bodies through: `open() do io`: ```julia HTTP.open("POST", "http://music.com/play") do io write(io, JSON.json([ "auth" => "12345XXXX", "song_id" => 7, ])) r = startread(io) @show r.status while !eof(io) bytes = readavailable(io) play_audio(bytes) end end ``` """ function request(method, url, h=nothing, b=nobody; headers=h, body=b, query=nothing, observelayers::Bool=false, kw...)::Response return request(HTTP.stack(observelayers), method, url, headers, body, query; kw...) end # layers are applied from left to right, i.e. the first layer is the outermost that is called first, which then calls into the second layer, etc. const STREAM_LAYERS = [timeoutlayer, exceptionlayer] const REQUEST_LAYERS = [redirectlayer, headerslayer, cookielayer, retrylayer] """ Layer Abstract type to represent a client-side middleware that exists for documentation purposes. A layer is any function of the form `f(::Handler) -> Handler`, where [`Handler`](@ref) is a function of the form `f(::Request) -> Response`. Note that the `Handler` definition is from the server-side documentation, and is "hard-coded" on the client side. It may also be apparent that a `Layer` is the same as the [`Middleware`](@ref) interface from server-side, which is true, but we define `Layer` to clarify the client-side distinction and its unique usage. Custom layers can be deployed in one of two ways: * [`HTTP.@client`](@ref): Create a custom "client" with shorthand verb definitions, but which include custom layers; only these new verb methods will use the custom layers. * [`HTTP.pushlayer!`](@ref)/[`HTTP.poplayer!`](@ref): Allows globally adding and removing layers from the default HTTP.jl layer stack; *all* http requests will then use the custom layers ### Quick Examples ```julia module Auth using HTTP function auth_layer(handler) # returns a `Handler` function; check for a custom keyword arg `authcreds` that # a user would pass like `HTTP.get(...; authcreds=creds)`. # We also accept trailing keyword args `kw...` and pass them along later. return function(req; authcreds=nothing, kw...) # only apply the auth layer if the user passed `authcreds` if authcreds !== nothing # we add a custom header with stringified auth creds HTTP.setheader(req, "X-Auth-Creds" => string(authcreds)) end # pass the request along to the next layer by calling `auth_layer` arg `handler` # also pass along the trailing keyword args `kw...` return handler(req; kw...) end end # Create a new client with the auth layer added HTTP.@client [auth_layer] end # module # Can now use custom client like: Auth.get(url; authcreds=creds) # performs GET request with auth_layer layer included # Or can include layer globally in all HTTP.jl requests HTTP.pushlayer!(Auth.auth_layer) # Now can use normal HTTP.jl methods and auth_layer will be included HTTP.get(url; authcreds=creds) ``` """ abstract type Layer end """ HTTP.pushlayer!(layer; request=true) Push a layer onto the stack of layers that will be applied to all requests. The "layer" is expected to be a function that takes and returns a `Handler` function. See [`Layer`](@ref) for more details. If `request=false`, the layer is expected to take and return a "stream" handler function. The custom `layer` will be put on the top of the stack, so it will be the first layer executed. To add a layer at the bottom of the stack, see [`HTTP.pushfirstlayer!`](@ref). """ pushlayer!(layer; request::Bool=true) = push!(request ? REQUEST_LAYERS : STREAM_LAYERS, layer) """ HTTP.pushfirstlayer!(layer; request=true) Push a layer to the start of the stack of layers that will be applied to all requests. The "layer" is expected to be a function that takes and returns a `Handler` function. See [`Layer`](@ref) for more details. If `request=false`, the layer is expected to take and return a "stream" handler function. The custom `layer` will be put on the bottom of the stack, so it will be the last layer executed. To add a layer at the top of the stack, see [`HTTP.pushlayer!`](@ref). """ pushfirstlayer!(layer; request::Bool=true) = pushfirst!(request ? REQUEST_LAYERS : STREAM_LAYERS, layer) """ HTTP.poplayer!(; request=true) Inverse of [`HTTP.pushlayer!`](@ref), removes the top layer of the global HTTP.jl layer stack. Can be used to "cleanup" after a custom layer has been added. If `request=false`, will remove the top "stream" layer as opposed to top "request" layer. """ poplayer!(; request::Bool=true) = pop!(request ? REQUEST_LAYERS : STREAM_LAYERS) """ HTTP.popfirstlayer!(; request=true) Inverse of [`HTTP.pushfirstlayer!`](@ref), removes the bottom layer of the global HTTP.jl layer stack. Can be used to "cleanup" after a custom layer has been added. If `request=false`, will remove the bottom "stream" layer as opposed to bottom "request" layer. """ popfirstlayer!(; request::Bool=true) = popfirst!(request ? REQUEST_LAYERS : STREAM_LAYERS) function stack( observelayers::Bool=false, # custom layers requestlayers=(), streamlayers=()) obs = observelayers ? observelayer : identity # stream layers if streamlayers isa NamedTuple inner_stream_layers = haskey(streamlayers, :last) ? streamlayers.last : () outer_stream_layers = haskey(streamlayers, :first) ? streamlayers.first : () else inner_stream_layers = streamlayers outer_stream_layers = () end layers = foldr((x, y) -> obs(x(y)), inner_stream_layers, init=obs(streamlayer)) layers2 = foldr((x, y) -> obs(x(y)), STREAM_LAYERS, init=layers) if !isempty(outer_stream_layers) layers2 = foldr((x, y) -> obs(x(y)), outer_stream_layers, init=layers2) end # request layers # messagelayer must be the 1st/outermost layer to convert initial args to Request if requestlayers isa NamedTuple inner_request_layers = haskey(requestlayers, :last) ? requestlayers.last : () outer_request_layers = haskey(requestlayers, :first) ? requestlayers.first : () else inner_request_layers = requestlayers outer_request_layers = () end layers3 = foldr((x, y) -> obs(x(y)), inner_request_layers; init=obs(connectionlayer(layers2))) layers4 = foldr((x, y) -> obs(x(y)), REQUEST_LAYERS; init=layers3) if !isempty(outer_request_layers) layers4 = foldr((x, y) -> obs(x(y)), outer_request_layers, init=layers4) end return messagelayer(layers4) end function request(stack::Base.Callable, method, url, h=nothing, b=nobody, q=nothing; headers=h, body=b, query=q, kw...)::Response return stack(string(method), request_uri(url, query), headers, body; kw...) end macro remove_linenums!(expr) return esc(Base.remove_linenums!(expr)) end """ HTTP.@client requestlayers HTTP.@client requestlayers streamlayers HTTP.@client (first=requestlayers, last=requestlayers) (first=streamlayers, last=streamlayers) Convenience macro for creating a custom HTTP.jl client that will include custom layers when performing requests. It's common to want to define a custom [`Layer`](@ref) to enhance a specific category of requests, such as custom authentcation for a web API. Instead of affecting the global HTTP.jl request stack via [`HTTP.pushlayer!`](@ref), a custom wrapper client can be defined with convenient shorthand methods. See [`Layer`](@ref) for an example of defining a custom layer and creating a new client that includes the layer. Custom layer arguments can be provided as a collection of request or stream-based layers; alternatively, a NamedTuple with keys `first` and `last` can be provided with values being a collection of layers. The NamedTuple form provides finer control over the order in which the layers will be included in the default http layer stack: `first` request layers are executed before all other layers, `last` request layers are executed right before all stream layers, and similarly for stream layers. An empty collection can always be passed for request or stream layers when not needed. One use case for custom clients is to control the value of standard `HTTP.request` keyword arguments. This can be achieved by passing a `(first=[defaultkeywordlayer],)` where `defaultkeywordlayer` is defined like: ```julia defaultkeywordlayer(handler) = (req; kw...) -> handler(req; retry=false, redirect=false, kw...) ``` This client-side layer is basically a no-op as it doesn't modify the request at all, except that it hard-codes the value of the `retry` and `redirect` keyword arguments. When we pass this layer as `(first=[defaultkeywordlayer],)` this ensures this layer will be executed before all other layers, effectively over-writing the default and any user-provided keyword arguments for `retry` or `redirect`. """ macro client(requestlayers, streamlayers=[]) return @remove_linenums! esc(quote get(a...; kw...) = ($__source__; request("GET", a...; kw...)) put(a...; kw...) = ($__source__; request("PUT", a...; kw...)) post(a...; kw...) = ($__source__; request("POST", a...; kw...)) patch(a...; kw...) = ($__source__; request("PATCH", a...; kw...)) head(a...; kw...) = ($__source__; request("HEAD", a...; kw...)) delete(a...; kw...) = ($__source__; request("DELETE", a...; kw...)) open(f, a...; kw...) = ($__source__; request(a...; iofunction=f, kw...)) function request(method, url, h=HTTP.Header[], b=HTTP.nobody; headers=h, body=b, query=nothing, observelayers::Bool=false, kw...)::HTTP.Response $__source__ HTTP.request(HTTP.stack(observelayers, $requestlayers, $streamlayers), method, url, headers, body, query; kw...) end end) end """ HTTP.get(url [, headers]; ) -> HTTP.Response Shorthand for `HTTP.request("GET", ...)`. See [`HTTP.request`](@ref). """ get(a...; kw...) = request("GET", a...; kw...) """ HTTP.put(url, headers, body; ) -> HTTP.Response Shorthand for `HTTP.request("PUT", ...)`. See [`HTTP.request`](@ref). """ put(a...; kw...) = request("PUT", a...; kw...) """ HTTP.post(url, headers, body; ) -> HTTP.Response Shorthand for `HTTP.request("POST", ...)`. See [`HTTP.request`](@ref). """ post(a...; kw...) = request("POST", a...; kw...) """ HTTP.patch(url, headers, body; ) -> HTTP.Response Shorthand for `HTTP.request("PATCH", ...)`. See [`HTTP.request`](@ref). """ patch(a...; kw...) = request("PATCH", a...; kw...) """ HTTP.head(url; ) -> HTTP.Response Shorthand for `HTTP.request("HEAD", ...)`. See [`HTTP.request`](@ref). """ head(u; kw...) = request("HEAD", u; kw...) """ HTTP.delete(url [, headers]; ) -> HTTP.Response Shorthand for `HTTP.request("DELETE", ...)`. See [`HTTP.request`](@ref). """ delete(a...; kw...) = request("DELETE", a...; kw...) request_uri(url, query) = URI(URI(url); query=query) request_uri(url, ::Nothing) = URI(url) """ HTTP.open(method, url, [,headers]) do io write(io, body) [startread(io) -> HTTP.Response] while !eof(io) readavailable(io) -> AbstractVector{UInt8} end end -> HTTP.Response The `HTTP.open` API allows the request body to be written to (and/or the response body to be read from) an `IO` stream. e.g. Streaming an audio file to the `vlc` player: ```julia HTTP.open(:GET, "https://tinyurl.com/bach-cello-suite-1-ogg") do http open(`vlc -q --play-and-exit --intf dummy -`, "w") do vlc write(vlc, http) end end ``` """ open(f::Function, method::Union{String,Symbol}, url, headers=Header[]; kw...)::Response = request(string(method), url, headers, nothing; iofunction=f, kw...) """ HTTP.openraw(method, url, [, headers])::Tuple{Connection, Response} Open a raw socket that is unmanaged by HTTP.jl. Useful for doing HTTP upgrades to other protocols. Any bytes of the body read from the socket when reading headers, is returned as excess bytes in the last tuple argument. Example of a WebSocket upgrade: ```julia headers = Dict( "Upgrade" => "websocket", "Connection" => "Upgrade", "Sec-WebSocket-Key" => "dGhlIHNhbXBsZSBub25jZQ==", "Sec-WebSocket-Version" => "13") socket, response, excess = HTTP.openraw("GET", "ws://echo.websocket.org", headers) # Write a WebSocket frame frame = UInt8[0x81, 0x85, 0x37, 0xfa, 0x21, 0x3d, 0x7f, 0x9f, 0x4d, 0x51, 0x58] write(socket, frame) ``` """ function openraw(method::Union{String,Symbol}, url, headers=Header[]; kw...)::Tuple{IO, Response} socketready = Channel{Tuple{IO, Response}}(0) Threads.@spawn HTTP.open(method, url, headers; kw...) do http HTTP.startread(http) socket = http.stream put!(socketready, (socket, http.message)) while(isopen(socket)) Base.wait_close(socket) end end take!(socketready) end """ parse(Request, str) parse(Response, str) Parse a string into a `Request` or `Response` object. """ function Base.parse(::Type{T}, str::AbstractString)::T where T <: Message buffer = Base.BufferStream() write(buffer, str) close(buffer) m = T() http = Stream(m, Connection(buffer)) m.body = read(http) closeread(http) return m end end # module 0/opt/julia/packages/HTTP/sJD5V/src/Conditions.jl module Conditions export @require, @ensure, precondition_error, postcondition_error import ..DEBUG_LEVEL # Get the calling function. See https://github.com/JuliaLang/julia/issues/6733 # (The macro form @__FUNCTION__ is hard to escape correctly, so just us a function.) function _funcname_expr() return :($(esc(Expr(:isdefined, Symbol("#self#")))) ? nameof($(esc(Symbol("#self#")))) : nothing) end @noinline function precondition_error(msg, calling_funcname) calling_funcname = calling_funcname === nothing ? "unknown" : calling_funcname return ArgumentError("$calling_funcname() requires $msg") end """ @require precondition [message] Throw `ArgumentError` if `precondition` is false. """ macro require(condition, msg = "`$condition`") :(if ! $(esc(condition)) throw(precondition_error($(esc(msg)), $(_funcname_expr()))) end) end @noinline function postcondition_error(msg, calling_funcname, ls="", l="", rs="", r="") calling_funcname = calling_funcname === nothing ? "unknown" : calling_funcname msg = "$calling_funcname() failed to ensure $msg" if ls != "" msg = string(msg, "\n", ls, " = ", sprint(show, l), "\n", rs, " = ", sprint(show, r)) end return AssertionError(msg) end # Copied from stdlib/Test/src/Test.jl:get_test_result() iscondition(ex) = isa(ex, Expr) && ex.head == :call && length(ex.args) == 3 && first(string(ex.args[1])) != '.' && (!isa(ex.args[2], Expr) || ex.args[2].head != :...) && (!isa(ex.args[3], Expr) || ex.args[3].head != :...) && (ex.args[1] === :(==) || Base.operator_precedence(ex.args[1]) == Base.operator_precedence(:(==))) """ @ensure postcondition [message] Throw `ArgumentError` if `postcondition` is false. """ macro ensure(condition, msg = "`$condition`") if DEBUG_LEVEL[] < 0 return :() end if iscondition(condition) l,r = condition.args[2], condition.args[3] ls, rs = string(l), string(r) return quote if ! $(esc(condition)) # FIXME double-execution of condition l and r! throw(postcondition_error($(esc(msg)), $(_funcname_expr()), $ls, $(esc(l)), $rs, $(esc(r)))) end end end :(if ! $(esc(condition)) throw(postcondition_error($(esc(msg)), $(_funcname_expr()))) end) end end # module0/opt/julia/packages/HTTP/sJD5V/src/access_log.jl@doc raw""" logfmt"..." Parse an [NGINX-style log format string](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) and return a function mapping `(io::IO, http::HTTP.Stream) -> body` suitable for passing to [`HTTP.listen`](@ref) using the `access_log` keyword argument. The following variables are currently supported: - `$http_name`: arbitrary request header (with `-` replaced with `_`, e.g. `http_user_agent`) - `$sent_http_name`: arbitrary response header (with `-` replaced with `_`) - `$request`: the request line, e.g. `GET /index.html HTTP/1.1` - `$request_method`: the request method - `$request_uri`: the request URI - `$remote_addr`: client address - `$remote_port`: client port - `$remote_user`: user name supplied with the Basic authentication - `$server_protocol`: server protocol - `$time_iso8601`: local time in ISO8601 format - `$time_local`: local time in Common Log Format - `$status`: response status code - `$body_bytes_sent`: number of bytes in response body ## Examples ```julia logfmt"[$time_iso8601] \\"$request\\" $status" # [2021-05-01T12:34:40+0100] "GET /index.html HTTP/1.1" 200 logfmt"$remote_addr \\"$http_user_agent\\"" # 127.0.0.1 "curl/7.47.0" ``` """ macro logfmt_str(s) return logfmt_parser(s) end function logfmt_parser(s) s = String(s) vars = Symbol[] ex = Expr(:call, :print, :io) i = 1 while i <= lastindex(s) j = findnext(==('\$'), s, i) if j === nothing j = lastindex(s) push!(ex.args, String(s[i:j])) break end if j > i push!(ex.args, String(s[i:prevind(s, j)])) end sym, j = Meta.parse(s, nextind(s, j); greedy=false) e = symbol_mapping(sym) isa(e, Tuple) ? push!(ex.args, e...) : push!(ex.args, e) i = j end f = Expr(:->, Expr(:tuple, :io, :http), ex) return f end function symbol_mapping(s::Symbol) str = string(s) if (m = match(r"^http_(.+)$", str); m !== nothing) hdr = replace(String(m[1]), '_' => '-') :(HTTP.header(http.message, $hdr, "-")) elseif (m = match(r"^sent_http_(.+)$", str); m !== nothing) hdr = replace(String(m[1]), '_' => '-') :(HTTP.header(http.message.response, $hdr, "-")) elseif s === :remote_addr :(http.stream.peerip) elseif s === :remote_port :(http.stream.peerport) elseif s === :remote_user :("-") # TODO: find from Basic auth... elseif s === :time_iso8601 if !Sys.iswindows() :(Libc.strftime("%FT%T%z", time())) else # TODO: Libc.strftime doesn't seem to work properly on Windows # so format without timezone using Dates stdlib :(Dates.format(now(), dateformat"yyyy-mm-dd\THH:MM:SS")) end elseif s === :time_local if !Sys.iswindows() :(Libc.strftime("%d/%b/%Y:%H:%M:%S %z", time())) else # TODO: Libc.strftime doesn't seem to work properly on Windows # so format without timezone using Dates stdlib :(Dates.format(now(), dateformat"dd/u/yyyy:HH:MM:SS")) end elseif s === :request m = symbol_mapping(:request_method) t = symbol_mapping(:request_uri) p = symbol_mapping(:server_protocol) (m, " ", t, " ", p...) elseif s === :request_method :(http.message.method) elseif s === :request_uri :(http.message.target) elseif s === :server_protocol ("HTTP/", :(http.message.version.major), ".", :(http.message.version.minor)) elseif s === :status :(http.message.response.status) elseif s === :body_bytes_sent return :(max(0, http.nwritten)) else error("unknown variable in logfmt: $s") end end """ common_logfmt(io::IO, http::HTTP.Stream) Format a log message in the Common Log Format and write to `io`. """ const common_logfmt = logfmt"$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent" """ combined_logfmt(io::IO, http::HTTP.Stream) Format a log message in the Combined Log Format and write to `io`. """ const combined_logfmt = logfmt"$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\"" +/opt/julia/packages/HTTP/sJD5V/src/Pairs.jlmodule Pairs export defaultbyfirst, setbyfirst, getbyfirst, setkv, getkv, rmkv """ setbyfirst(collection, item) -> item Set `item` in a `collection`. If `first() of an exisiting item matches `first(item)` it is replaced. Otherwise the new `item` is inserted at the end of the `collection`. """ function setbyfirst(c, item, eq = ==) k = first(item) i = findfirst(x->eq(first(x), k), c) if i === nothing push!(c, item) else c[i] = item end return item end """ getbyfirst(collection, key [, default]) -> item Get `item` from collection where `first(item)` matches `key`. """ function getbyfirst(c, k, default=nothing, eq = ==) i = findfirst(x->eq(first(x), k), c) return i === nothing ? default : c[i] end """ defaultbyfirst(collection, item) If `first(item)` does not match match `first()` of any existing items, insert the new `item` at the end of the `collection`. """ function defaultbyfirst(c, item, eq = ==) k = first(item) if findfirst(x->eq(first(x), k), c) === nothing push!(c, item) end return end """ setkv(collection, key, value) Set `value` for `key` in collection of key/value `Pairs`. """ setkv(c, k, v) = setbyfirst(c, k => v) """ getkv(collection, key [, default]) -> value Get `value` for `key` in collection of key/value `Pairs`, where `first(item) == key` and `value = item[2]` """ function getkv(c, k, default=nothing) i = findfirst(x->first(x) == k, c) return i === nothing ? default : c[i][2] end """ rmkv(collection, key) Remove `key` from `collection` of key/value `Pairs`. """ function rmkv(c, k, eq = ==) i = findfirst(x->eq(first(x), k), c) if !(i === nothing) deleteat!(c, i) end return end end # module Pairs ./opt/julia/packages/HTTP/sJD5V/src/IOExtras.jl """ IOExtras This module defines extensions to the `Base.IO` interface to support: - `startwrite`, `closewrite`, `startread` and `closeread` for streams with transactional semantics. """ module IOExtras using Sockets using MbedTLS: SSLContext, MbedException using OpenSSL: SSLStream export bytes, isbytes, nbytes, nobytes, startwrite, closewrite, startread, closeread, readuntil, tcpsocket, localport, safe_getpeername """ bytes(x) If `x` is "castable" to an `AbstractVector{UInt8}`, then an `AbstractVector{UInt8}` is returned; otherwise `x` is returned. """ function bytes end bytes(s::AbstractVector{UInt8}) = s bytes(s::AbstractString) = codeunits(s) bytes(x) = x """whether `x` is "castable" to an `AbstractVector{UInt8}`; i.e. you can call `bytes(x)` if `isbytes(x)` === true""" isbytes(x) = x isa AbstractVector{UInt8} || x isa AbstractString """ nbytes(x) -> Int Length in bytes of `x` if `x` is `isbytes(x)`. """ function nbytes end nbytes(x) = nothing nbytes(x::AbstractVector{UInt8}) = length(x) nbytes(x::AbstractString) = sizeof(x) nbytes(x::Vector{T}) where T <: AbstractString = sum(sizeof, x) nbytes(x::Vector{T}) where T <: AbstractVector{UInt8} = sum(length, x) nbytes(x::IOBuffer) = bytesavailable(x) nbytes(x::Vector{IOBuffer}) = sum(bytesavailable, x) _doc = """ startwrite(::IO) closewrite(::IO) startread(::IO) closeread(::IO) Signal start/end of write or read operations. """ @static if isdefined(Base, :startwrite) "$_doc" Base.startwrite(io) = nothing else "$_doc" startwrite(io) = nothing end @static if isdefined(Base, :closewrite) "$_doc" Base.closewrite(io) = nothing else "$_doc" closewrite(io) = nothing end @static if isdefined(Base, :startread) "$_doc" Base.startread(io) = nothing else "$_doc" startread(io) = nothing end @static if isdefined(Base, :closeread) "$_doc" Base.closeread(io) = nothing else "$_doc" closeread(io) = nothing end tcpsocket(io::SSLContext)::TCPSocket = io.bio tcpsocket(io::SSLStream)::TCPSocket = io.io tcpsocket(io::TCPSocket)::TCPSocket = io localport(io) = try !isopen(tcpsocket(io)) ? 0 : Sockets.getsockname(tcpsocket(io))[2] catch 0 end function safe_getpeername(io) try if isopen(tcpsocket(io)) return Sockets.getpeername(tcpsocket(io)) end catch end return IPv4(0), UInt16(0) end const nobytes = view(UInt8[], 1:0) readuntil(args...) = Base.readuntil(args...) """ Read from an `IO` stream until `find_delimiter(bytes)` returns non-zero. Return view of bytes up to the delimiter. """ function readuntil(buf::IOBuffer, find_delimiter::F #= Vector{UInt8} -> Int =# ) where {F <: Function} l = find_delimiter(view(buf.data, buf.ptr:buf.size)) if l == 0 return nobytes end bytes = buf.data[buf.ptr:buf.ptr + l - 1] buf.ptr += l return bytes end end -/opt/julia/packages/HTTP/sJD5V/src/Strings.jlAmodule Strings export HTTPVersion, escapehtml, tocameldash, iso8859_1_to_utf8, ascii_lc_isequal using ..IOExtras # A `Base.VersionNumber` is a SemVer spec, whereas a HTTP versions is just 2 digits, # This allows us to use a smaller type and more importantly write a simple parse method # that avoid allocations. """ HTTPVersion(major, minor) The HTTP version number consists of two digits separated by a "." (period or decimal point). The first digit (`major` version) indicates the HTTP messaging syntax, whereas the second digit (`minor` version) indicates the highest minor version within that major version to which the sender is conformant and able to understand for future communication. See [RFC7230 2.6](https://tools.ietf.org/html/rfc7230#section-2.6) """ struct HTTPVersion major::UInt8 minor::UInt8 end HTTPVersion(major::Integer) = HTTPVersion(major, 0x00) HTTPVersion(v::AbstractString) = parse(HTTPVersion, v) HTTPVersion(v::VersionNumber) = convert(HTTPVersion, v) # Lossy conversion. We ignore patch/prerelease/build parts even if non-zero/non-empty, # because we don't want to add overhead for a case that should never be relevant. Base.convert(::Type{HTTPVersion}, v::VersionNumber) = HTTPVersion(v.major, v.minor) Base.VersionNumber(v::HTTPVersion) = VersionNumber(v.major, v.minor) Base.show(io::IO, v::HTTPVersion) = print(io, "HTTPVersion(\"", string(v.major), ".", string(v.minor), "\")") Base.:(==)(va::VersionNumber, vb::HTTPVersion) = va == VersionNumber(vb) Base.:(==)(va::HTTPVersion, vb::VersionNumber) = VersionNumber(va) == vb Base.isless(va::VersionNumber, vb::HTTPVersion) = isless(va, VersionNumber(vb)) Base.isless(va::HTTPVersion, vb::VersionNumber) = isless(VersionNumber(va), vb) function Base.isless(va::HTTPVersion, vb::HTTPVersion) va.major < vb.major && return true va.major > vb.major && return false va.minor < vb.minor && return true return false end function Base.parse(::Type{HTTPVersion}, v::AbstractString) ver = tryparse(HTTPVersion, v) ver === nothing && throw(ArgumentError("invalid HTTP version string: $(repr(v))")) return ver end # We only support single-digits for major and minor versions # - we can parse 0.9 but not 0.10 # - we can parse 9.0 but not 10.0 function Base.tryparse(::Type{HTTPVersion}, v::AbstractString) isempty(v) && return nothing len = ncodeunits(v) i = firstindex(v) d1 = v[i] if isdigit(d1) major = parse(UInt8, d1) else return nothing end i = nextind(v, i) i > len && return HTTPVersion(major) dot = v[i] dot == '.' || return nothing i = nextind(v, i) i > len && return HTTPVersion(major) d2 = v[i] if isdigit(d2) minor = parse(UInt8, d2) else return nothing end return HTTPVersion(major, minor) end """ escapehtml(i::String) Returns a string with special HTML characters escaped: &, <, >, ", ' """ function escapehtml(i::AbstractString) # Refer to http://stackoverflow.com/a/7382028/3822752 for spec. links o = replace(i, "&" =>"&") o = replace(o, "\""=>""") o = replace(o, "'" =>"'") o = replace(o, "<" =>"<") o = replace(o, ">" =>">") return o end """ tocameldash(s::String) Ensure the first character and characters that follow a '-' are uppercase. """ function tocameldash(s::String) toUpper = UInt8('A') - UInt8('a') v = Vector{UInt8}(bytes(s)) upper = true for i = 1:length(v) @inbounds b = v[i] if upper islower(b) && (v[i] = b + toUpper) else isupper(b) && (v[i] = lower(b)) end upper = b == UInt8('-') end return String(v) end tocameldash(s::AbstractString) = tocameldash(String(s)) @inline islower(b::UInt8) = UInt8('a') <= b <= UInt8('z') @inline isupper(b::UInt8) = UInt8('A') <= b <= UInt8('Z') @inline lower(c::UInt8) = c | 0x20 """ iso8859_1_to_utf8(bytes::AbstractVector{UInt8}) Convert from ISO8859_1 to UTF8. """ function iso8859_1_to_utf8(bytes::AbstractVector{UInt8}) io = IOBuffer() for b in bytes if b < 0x80 write(io, b) else write(io, 0xc0 | (b >> 6)) write(io, 0x80 | (b & 0x3f)) end end return String(take!(io)) end """ Convert ASCII (RFC20) character `c` to lower case. """ ascii_lc(c::UInt8) = c in UInt8('A'):UInt8('Z') ? c + 0x20 : c """ Case insensitive ASCII character comparison. """ ascii_lc_isequal(a::UInt8, b::UInt8) = ascii_lc(a) == ascii_lc(b) """ HTTP.ascii_lc_isequal(a::String, b::String) Case insensitive ASCII string comparison. """ function ascii_lc_isequal(a, b) acu = codeunits(a) bcu = codeunits(b) len = length(acu) len != length(bcu) && return false for i = 1:len @inbounds !ascii_lc_isequal(acu[i], bcu[i]) && return false end return true end end # module Strings 0/opt/julia/packages/HTTP/sJD5V/src/Exceptions.jl% module Exceptions export @try, HTTPError, ConnectError, TimeoutError, StatusError, RequestError, current_exceptions_to_string using LoggingExtras, ExceptionUnwrapping import ..HTTP # for doc references @eval begin """ @try Permitted Error Types expr Convenience macro for wrapping an expression in a try/catch block where thrown exceptions are ignored. """ macro $(:try)(exes...) errs = Any[exes...] ex = pop!(errs) isempty(errs) && error("no permitted errors") quote try $(esc(ex)) catch e e isa InterruptException && rethrow(e) |($([:(e isa $(esc(err))) for err in errs]...)) || rethrow(e) end end end end # @eval abstract type HTTPError <: Exception end """ HTTP.ConnectError Raised when an error occurs while trying to establish a request connection to the remote server. To see the underlying error, see the `error` field. """ struct ConnectError <: HTTPError url::String # the URL of the request error::Any # underlying error end ExceptionUnwrapping.unwrap_exception(e::ConnectError) = e.error function Base.showerror(io::IO, e::ConnectError) print(io, "HTTP.ConnectError for url = `$(e.url)`: ") Base.showerror(io, e.error) end """ HTTP.TimeoutError Raised when a request times out according to `readtimeout` keyword argument provided. """ struct TimeoutError <: HTTPError readtimeout::Int end Base.showerror(io::IO, e::TimeoutError) = print(io, "TimeoutError: Connection closed after $(e.readtimeout) seconds") """ HTTP.StatusError Raised when an `HTTP.Response` has a `4xx`, `5xx` or unrecognised status code. Fields: - `status::Int16`, the response status code. - `method::String`, the request method. - `target::String`, the request target. - `response`, the [`HTTP.Response`](@ref) """ struct StatusError <: HTTPError status::Int16 method::String target::String response::Any end """ HTTP.RequestError Raised when an error occurs while physically sending a request to the remote server or reading the response back. To see the underlying error, see the `error` field. """ struct RequestError <: HTTPError request::Any error::Any end ExceptionUnwrapping.unwrap_exception(e::RequestError) = e.error function Base.showerror(io::IO, e::RequestError) println(io, "HTTP.RequestError:") println(io, "HTTP.Request:") Base.show(io, e.request) println(io, "Underlying error:") Base.showerror(io, e.error) end function current_exceptions_to_string() buf = IOBuffer() println(buf) println(buf, "\n===========================\nHTTP Error message:\n") exc = @static if VERSION >= v"1.8.0-" Base.current_exceptions() else Base.catch_stack() end Base.display_error(buf, exc) return String(take!(buf)) end end # module Exceptions +/opt/julia/packages/HTTP/sJD5V/src/sniff.jl-module Sniff export sniff, isjson const CodeUnits = Union{Vector{UInt8}, Base.CodeUnits} # compression detection const ZIP = UInt8[0x50, 0x4b, 0x03, 0x04] const GZIP = UInt8[0x1f, 0x8b, 0x08] iscompressed(bytes::CodeUnits) = length(bytes) > 3 && (all(bytes[1:4] .== ZIP) || all(bytes[1:3] .== GZIP)) iscompressed(str::String) = iscompressed(codeunits(str)) iscompressed(d::Dict) = false iscompressed(d) = false # Based on the net/http/sniff.go implementation of DetectContentType # sniff implements the algorithm described # at http://mimesniff.spec.whatwg.org/ to determine the # Content-Type of the given data. It considers at most the # first 512 bytes of data. sniff always returns # a valid MIME type: if it cannot determine a more specific one, it # returns "application/octet-stream". const MAXSNIFFLENGTH = 512 const WHITESPACE = Set{UInt8}([UInt8('\t'),UInt8('\n'),UInt8('\u0c'),UInt8('\r'),UInt8(' ')]) """ HTTP.sniff(content::Union{Vector{UInt8}, String, IO}) => String (mimetype) `HTTP.sniff` will look at the first 512 bytes of `content` to try and determine a valid mimetype. If a mimetype can't be determined appropriately, `"application/octet-stream"` is returned. Supports JSON detection through the `HTTP.isjson(content)` function. ## Examples ```julia julia> HTTP.sniff("Hello world!!") "text/plain; charset=utf-8" julia> HTTP.sniff("Hello world!!") "text/html; charset=utf-8" julia> HTTP.sniff("{\"a\": -1.0}") "application/json; charset=utf-8" ``` """ function sniff end function sniff(body::IO) alreadymarked = ismarked(body) mark(body) data = read(body, MAXSNIFFLENGTH) reset(body) alreadymarked && mark(body) return sniff(data) end sniff(str::String) = sniff(codeunits(str)[1:min(length(codeunits(str)), MAXSNIFFLENGTH)]) function sniff(data::CodeUnits) firstnonws = 1 while firstnonws < length(data) && data[firstnonws] in WHITESPACE firstnonws += 1 end for sig in SNIFF_SIGNATURES ismatch(sig, data, firstnonws) && return contenttype(sig) end return "application/octet-stream" # fallback end struct Exact sig::CodeUnits contenttype::String end contenttype(e::Exact) = e.contenttype function ismatch(e::Exact, data::CodeUnits, firstnonws) length(data) < length(e.sig) && return false for i = eachindex(e.sig) e.sig[i] == data[i] || return false end return true end struct Masked mask::CodeUnits pat::CodeUnits skipws::Bool contenttype::String end Masked(mask::CodeUnits, pat::CodeUnits, contenttype::String) = Masked(mask, pat, false, contenttype) contenttype(m::Masked) = m.contenttype function ismatch(m::Masked, data::CodeUnits, firstnonws) # pattern matching algorithm section 6 # https://mimesniff.spec.whatwg.org/#pattern-matching-algorithm sk = (m.skipws ? firstnonws : 1) - 1 length(m.pat) != length(m.mask) && return false length(data) < length(m.mask) && return false for (i, mask) in enumerate(m.mask) (data[i+sk] & mask) != m.pat[i] && return false end return true end struct HTMLSig html::CodeUnits HTMLSig(str::String) = new(codeunits(str)) end contenttype(h::HTMLSig) = "text/html; charset=utf-8" function ismatch(h::HTMLSig, data::CodeUnits, firstnonws) length(data) < length(h.html)+1 && return false for (i, b) in enumerate(h.html) db = data[i+firstnonws-1] (UInt8('A') <= b && b <= UInt8('Z')) && (db &= 0xDF) b != db && return false end data[length(h.html)+firstnonws] in (UInt8(' '), UInt8('>')) || return false return true end struct MP4Sig end contenttype(::Type{MP4Sig}) = "video/mp4" function byteequal(data1, ind, data2) for i = eachindex(data2) @inbounds data1[ind+i-1] == data2[i] || return false end return true end const mp4ftype = codeunits("ftyp") const mp4 = codeunits("mp4") # Byte swap int bigend(b) = UInt32(b[4]) | UInt32(b[3])<<8 | UInt32(b[2])<<16 | UInt32(b[1])<<24 function ismatch(::Type{MP4Sig}, data::CodeUnits, firstnonws) # https://mimesniff.spec.whatwg.org/#signature-for-mp4 # c.f. section 6.2.1 length(data) < 12 && return false boxsize = Int(bigend(data)) (boxsize % 4 != 0 || length(data) < boxsize) && return false byteequal(data, 5, mp4ftype) || return false for st = 9:4:boxsize+1 st == 13 && continue byteequal(data, st, mp4) && return true end return false end struct TextSig end contenttype(::Type{TextSig}) = "text/plain; charset=utf-8" function ismatch(::Type{TextSig}, data::CodeUnits, firstnonws) # c.f. section 5, step 4. for i = firstnonws:min(length(data),MAXSNIFFLENGTH) b = data[i] (b <= 0x08 || b == 0x0B || 0x0E <= b <= 0x1A || 0x1C <= b <= 0x1F) && return false end return true end struct JSONSig end contenttype(::Type{JSONSig}) = "application/json; charset=utf-8" ismatch(::Type{JSONSig}, data::CodeUnits, firstnonws) = isjson(data)[1] const DISPLAYABLE_TYPES = ["text/html; charset=utf-8", "text/plain; charset=utf-8", "application/json; charset=utf-8", "text/xml; charset=utf-8", "text/plain; charset=utf-16be", "text/plain; charset=utf-16le"] # Data matching the table in section 6. const SNIFF_SIGNATURES = [ HTMLSig("= maxlen && return true, 0x00, i @inbounds b = bytes[i] return false, b, i end function restofstring(bytes, i, maxlen) while true eof, b, i = nextbyte(bytes, i, maxlen) eof && return i b == DOUBLE_QUOTE && return i if b == ESCAPE eof, b, i = nextbyte(bytes, i, maxlen) end end end macro expect(ch) return esc(quote eof, b, i = ignorewhitespace(bytes, i, maxlen) eof && return true, i b == $ch || return false, i end) end const OPEN_CURLY_BRACE = UInt8('{') const CLOSE_CURLY_BRACE = UInt8('}') const OPEN_SQUARE_BRACE = UInt8('[') const CLOSE_SQUARE_BRACE = UInt8(']') const DOUBLE_QUOTE = UInt8('"') const ESCAPE = UInt8('\\') const COMMA = UInt8(',') const COLON = UInt8(':') const MINUS = UInt8('-') const ZERO = UInt8('0') const NINE = UInt8('9') const LITTLE_N = UInt8('n') const LITTLE_U = UInt8('u') const LITTLE_L = UInt8('l') const LITTLE_T = UInt8('t') const LITTLE_R = UInt8('r') const LITTLE_E = UInt8('e') const LITTLE_F = UInt8('f') const LITTLE_A = UInt8('a') const LITTLE_S = UInt8('s') const PERIOD = UInt8('.') const REF = Vector{Ptr{UInt8}}(undef, 1) function isjson(bytes, i=0, maxlen=min(length(bytes), MAXSNIFFLENGTH)) # ignore leading whitespace isempty(bytes) && return false, 0 eof, b, i = ignorewhitespace(bytes, i, maxlen) eof && return true, i # must start with: if b == OPEN_CURLY_BRACE # '{' start of object # must then read a string key, potential whitespace, then colon, potential whitespace then recursively check `isjson` while true @expect DOUBLE_QUOTE i = restofstring(bytes, i, maxlen) @expect COLON ret, i = isjson(bytes, i, maxlen) ret || return false, i eof, b, i = ignorewhitespace(bytes, i, maxlen) (eof || b == CLOSE_CURLY_BRACE) && return true, i b != COMMA && return false, i end elseif b == OPEN_SQUARE_BRACE # '[' start of array # peek at next byte to check for empty array ia = i eof, b, i = nextbyte(bytes, i, maxlen) if b != CLOSE_SQUARE_BRACE i = ia # recursively check `isjson`, then potential whitespace, then ',' or ']' while true ret, i = isjson(bytes, i, maxlen) ret || return false, i eof, b, i = ignorewhitespace(bytes, i, maxlen) (eof || b == CLOSE_SQUARE_BRACE) && return true, i b != COMMA && return false, i end end elseif b == DOUBLE_QUOTE # '"' start of string # must read until end of string w/ potential escaped '"' i = restofstring(bytes, i, maxlen) elseif (ZERO <= b <= NINE) || (b == MINUS) # must read until end of number v = zero(Float64) ptr = pointer(bytes) + i - 1 v = ccall(:jl_strtod_c, Float64, (Ptr{UInt8}, Ptr{Ptr{UInt8}}), ptr, REF) i += REF[1] - ptr - 1 elseif b == LITTLE_N # null @expect LITTLE_U @expect LITTLE_L @expect LITTLE_L elseif b == LITTLE_T # true @expect LITTLE_R @expect LITTLE_U @expect LITTLE_E elseif b == LITTLE_F # false @expect LITTLE_A @expect LITTLE_L @expect LITTLE_S @expect LITTLE_E else return false, i end return true, i end end # module//opt/julia/packages/HTTP/sJD5V/src/multipart.jlmodule Forms export Form, Multipart, content_type using ..IOExtras, ..Sniff, ..Conditions import ..HTTP # for doc references # Form request body mutable struct Form <: IO data::Vector{IO} index::Int mark::Int boundary::String end Form(f::Form) = f Base.eof(f::Form) = f.index > length(f.data) Base.isopen(f::Form) = false Base.close(f::Form) = nothing Base.length(f::Form) = sum(x->isa(x, IOStream) ? filesize(x) - position(x) : bytesavailable(x), f.data) IOExtras.nbytes(x::Form) = length(x) function Base.mark(f::Form) foreach(mark, f.data) f.mark = f.index return end function Base.reset(f::Form) foreach(reset, f.data) f.index = f.mark f.mark = -1 return end function Base.unmark(f::Form) foreach(unmark, f.data) f.mark = -1 return end function Base.position(f::Form) index = f.index foreach(mark, f.data) return index end function Base.seek(f::Form, pos) f.index = pos foreach(reset, f.data) return end Base.readavailable(f::Form) = read(f) function Base.read(f::Form) result = UInt8[] for io in f.data append!(result, read(io)) end f.index = length(f.data) + 1 return result end function Base.read(f::Form, n::Integer) nb = 0 result = UInt8[] while nb < n d = read(f.data[f.index], n - nb) nb += length(d) append!(result, d) eof(f.data[f.index]) && (f.index += 1) f.index > length(f.data) && break end return result end """ HTTP.Form(data; boundary=string(rand(UInt128), base=16)) Construct a request body for multipart/form-data encoding from `data`. `data` must iterate key-value pairs (e.g. `AbstractDict` or `Vector{Pair}`) where the key/value of the iterator is the key/value of each mutipart boundary chunk. Files and other large data arguments can be provided as values as IO arguments: either an `IOStream` such as returned via `open(file)`, or an `IOBuffer` for in-memory data. For complete control over a multipart chunk's details, an [`HTTP.Multipart`](@ref) type is provided to support setting the `filename`, `Content-Type`, and `Content-Transfer-Encoding`. # Examples ```julia data = Dict( "text" => "text data", # filename (cat.png) and content-type (image/png) inferred from the IOStream "file1" => open("cat.png"), # manully controlled chunk "file2" => HTTP.Multipart("dog.jpeg", open("mydog.jpg"), "image/jpeg"), ) body = HTTP.Form(data) headers = [] HTTP.post(url, headers, body) ``` """ function Form(d; boundary=string(rand(UInt128), base=16)) # https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html bcharsnospace = raw"\w'\(\)\+,-\./:=\?" boundary_re = Regex("^[$bcharsnospace ]{0,69}[$bcharsnospace]\$") @require match(boundary_re, boundary) !== nothing @require eltype(d) <: Pair data = IO[] io = IOBuffer() len = length(d) for (i, (k, v)) in enumerate(d) write(io, (i == 1 ? "" : "\r\n") * "--" * boundary * "\r\n") write(io, "Content-Disposition: form-data; name=\"$k\"") if isa(v, IO) writemultipartheader(io, v) seekstart(io) push!(data, io) push!(data, v) io = IOBuffer() else write(io, "\r\n\r\n") write(io, v) end end # write final boundary write(io, "\r\n--" * boundary * "--" * "\r\n") seekstart(io) push!(data, io) return Form(data, 1, -1, boundary) end function writemultipartheader(io::IOBuffer, i::IOStream) write(io, "; filename=\"$(basename(i.name[7:end-1]))\"\r\n") write(io, "Content-Type: $(sniff(i))\r\n\r\n") return end function writemultipartheader(io::IOBuffer, i::IO) write(io, "\r\n\r\n") return end """ HTTP.Multipart(filename::String, data::IO, content_type=HTTP.sniff(data), content_transfer_encoding="") A type to represent a single multipart upload chunk for a file. This type would be used as the value in a key-value pair when constructing a [`HTTP.Form`](@ref) for a request body (see example below). The `data` argument must be an `IO` type such as `IOStream`, or `IOBuffer`. The `content_type` and `content_transfer_encoding` arguments allow manual setting of these multipart headers. `Content-Type` will default to the result of the `HTTP.sniff(data)` mimetype detection algorithm, whereas `Content-Transfer-Encoding` will be left out if not specified. # Examples ```julia body = HTTP.Form(Dict( "key" => HTTP.Multipart("File.txt", open("MyFile.txt"), "text/plain"), )) headers = [] HTTP.post(url, headers, body) ``` # Extended help Filename SHOULD be included when the Multipart represents the contents of a file [RFC7578 4.2](https://tools.ietf.org/html/rfc7578#section-4.2) Content-Disposition set to "form-data" MUST be included with each Multipart. An additional "name" parameter MUST be included An optional "filename" parameter SHOULD be included if the contents of a file are sent This will be formatted such as: Content-Disposition: form-data; name="user"; filename="myfile.txt" [RFC7578 4.2](https://tools.ietf.org/html/rfc7578#section-4.2) Content-Type for each Multipart is optional, but SHOULD be included if the contents of a file are sent. [RFC7578 4.4](https://tools.ietf.org/html/rfc7578#section-4.4) Content-Transfer-Encoding for each Multipart is deprecated [RFC7578 4.7](https://tools.ietf.org/html/rfc7578#section-4.7) Other Content- header fields MUST be ignored [RFC7578 4.8](https://tools.ietf.org/html/rfc7578#section-4.8) """ mutable struct Multipart{T <: IO} <: IO filename::Union{String, Nothing} data::T contenttype::String contenttransferencoding::String name::String end function Multipart(f::Union{AbstractString, Nothing}, data::T, ct::AbstractString="", cte::AbstractString="", name::AbstractString="") where {T<:IO} f = f !== nothing ? String(f) : nothing return Multipart{T}(f, data, String(ct), String(cte), String(name)) end function Base.show(io::IO, m::Multipart{T}) where {T} items = ["data=::$T", "contenttype=\"$(m.contenttype)\"", "contenttransferencoding=\"$(m.contenttransferencoding)\")"] m.filename === nothing || pushfirst!(items, "filename=\"$(m.filename)\"") print(io, "HTTP.Multipart($(join(items, ", ")))") end Base.bytesavailable(m::Multipart{T}) where {T} = isa(m.data, IOStream) ? filesize(m.data) - position(m.data) : bytesavailable(m.data) Base.eof(m::Multipart{T}) where {T} = eof(m.data) Base.read(m::Multipart{T}, n::Integer) where {T} = read(m.data, n) Base.read(m::Multipart{T}) where {T} = read(m.data) Base.mark(m::Multipart{T}) where {T} = mark(m.data) Base.reset(m::Multipart{T}) where {T} = reset(m.data) Base.seekstart(m::Multipart{T}) where {T} = seekstart(m.data) function writemultipartheader(io::IOBuffer, i::Multipart) if i.filename === nothing write(io, "\r\n") else write(io, "; filename=\"$(i.filename)\"\r\n") end contenttype = i.contenttype == "" ? sniff(i.data) : i.contenttype write(io, "Content-Type: $(contenttype)\r\n") write(io, i.contenttransferencoding == "" ? "\r\n" : "Content-Transfer-Encoding: $(i.contenttransferencoding)\r\n\r\n") return end content_type(f::Form) = "Content-Type" => "multipart/form-data; boundary=$(f.boundary)" end # module-/opt/julia/packages/HTTP/sJD5V/src/Parsers.jl,""" The parser separates a raw HTTP Message into its component parts. If the input data is invalid the Parser throws a `HTTP.ParseError`. The `parse_*` functions processes a single element of a HTTP Message at a time and return a `SubString` containing the unused portion of the input. The Parser does not interpret the Message Headers. It is beyond the scope of the Parser to deal with repeated header fields, multi-line values, cookies or case normalization. The Parser has no knowledge of the high-level `Request` and `Response` structs defined in `Messages.jl`. However, the `Request` and `Response` structs must have field names compatible with those expected by the `parse_status_line!` and `parse_request_line!` functions. """ module Parsers import ..access_threaded using ..Strings export Header, Headers, find_end_of_header, find_end_of_chunk_size, find_end_of_trailer, parse_status_line!, parse_request_line!, parse_header_field, parse_chunk_size, ParseError include("parseutils.jl") const emptyss = SubString("",1,0) const emptyheader = emptyss => emptyss const Header = Pair{SubString{String},SubString{String}} const Headers = Vector{Header} """ ParseError <: Exception Parser input was invalid. Fields: - `code`, error code - `bytes`, the offending input. """ struct ParseError <: Exception code::Symbol bytes::SubString{String} end ParseError(code::Symbol, bytes="") = ParseError(code, first(split(String(bytes), '\n'))) # Regular expressions for parsing HTTP start-line and header-fields init!(r::RegexAndMatchData) = (Base.compile(r.re); initialize!(r); r) """ https://tools.ietf.org/html/rfc7230#section-3.1.1 request-line = method SP request-target SP HTTP-version CRLF """ const request_line_regex = RegexAndMatchData[] function request_line_regex_f() r = RegexAndMatchData(r"""^ (?: \r? \n) ? # ignore leading blank line ([!#$%&'*+\-.^_`|~[:alnum:]]+) [ ]+ # 1. method = token (RFC7230 3.2.6) ([^.][^ \r\n]*) [ ]+ # 2. target HTTP/(\d\.\d) # 3. version \r? \n # CRLF """x) init!(r) end """ https://tools.ietf.org/html/rfc7230#section-3.1.2 status-line = HTTP-version SP status-code SP reason-phrase CRLF See: [#190](https://github.com/JuliaWeb/HTTP.jl/issues/190#issuecomment-363314009) """ const status_line_regex = RegexAndMatchData[] function status_line_regex_f() r = RegexAndMatchData(r"""^ [ ]? # Issue #190 HTTP/(\d\.\d) [ ]+ # 1. version (\d\d\d) .* # 2. status \r? \n # CRLF """x) init!(r) end """ https://tools.ietf.org/html/rfc7230#section-3.2 header-field = field-name ":" OWS field-value OWS """ const header_field_regex = RegexAndMatchData[] function header_field_regex_f() r = RegexAndMatchData(r"""^ ([!#$%&'*+\-.^_`|~[:alnum:]]+) : # 1. field-name = token (RFC7230 3.2.6) [ \t]* # OWS ([^\r\n]*?) # 2. field-value [ \t]* # OWS \r? \n # CRLF (?= [^ \t]) # no WS on next line """x) init!(r) end """ https://tools.ietf.org/html/rfc7230#section-3.2.4 obs-fold = CRLF 1*( SP / HTAB ) """ const obs_fold_header_field_regex = RegexAndMatchData[] function obs_fold_header_field_regex_f() r = RegexAndMatchData(r"""^ ([!#$%&'*+\-.^_`|~[:alnum:]]+) : # 1. field-name = token (RFC7230 3.2.6) [ \t]* # OWS ([^\r\n]* # 2. field-value (?: \r? \n [ \t] [^\r\n]*)*) # obs-fold [ \t]* # OWS \r? \n # CRLF """x) init!(r) end const empty_header_field_regex = RegexAndMatchData[] function empty_header_field_regex_f() r = RegexAndMatchData(r"^ \r? \n"x) init!(r) end # HTTP start-line and header-field parsing """ Arbitrary limit to protect against denial of service attacks. """ const header_size_limit = Int(0x10000) """ find_end_of_header(bytes) -> length or 0 Find length of header delimited by `\\r\\n\\r\\n` or `\\n\\n`. """ function find_end_of_header(bytes::AbstractVector{UInt8}; allow_obs_fold=true) buf = 0xFFFFFFFF l = min(length(bytes), header_size_limit) i = 1 while i <= l @inbounds x = bytes[i] if x == 0x0D || x == 0x0A buf = (buf << 8) | UInt32(x) # "Although the line terminator for the start-line and header # fields is the sequence CRLF, a recipient MAY recognize a single # LF as a line terminator" # [RFC7230 3.5](https://tools.ietf.org/html/rfc7230#section-3.5) buf16 = buf & 0xFFFF if buf == 0x0D0A0D0A || buf16 == 0x0A0A return i end # "A server that receives an obs-fold ... MUST either reject the # message by sending a 400 (Bad Request) ... or replace each # received obs-fold with one or more SP octets..." # [RFC7230 3.2.4](https://tools.ietf.org/html/rfc7230#section-3.2.4) if !allow_obs_fold && (buf16 == 0x0A20 || buf16 == 0x0A09) throw(ParseError(:HEADER_CONTAINS_OBS_FOLD, bytes)) end else buf = 0xFFFFFFFF end i += 1 end if i > header_size_limit throw(ParseError(:HEADER_SIZE_EXCEEDS_LIMIT)) end return 0 end """ Parse HTTP request-line `bytes` and set the `method`, `target` and `version` fields of `request`. Return a `SubString` containing the header-field lines. """ function parse_request_line!(bytes::AbstractString, request)::SubString{String} re = access_threaded(request_line_regex_f, request_line_regex) if !exec(re, bytes) throw(ParseError(:INVALID_REQUEST_LINE, bytes)) end request.method = group(1, re, bytes) request.target = group(2, re, bytes) request.version = HTTPVersion(group(3, re, bytes)) return nextbytes(re, bytes) end """ Parse HTTP response-line `bytes` and set the `status` and `version` fields of `response`. Return a `SubString` containing the header-field lines. """ function parse_status_line!(bytes::AbstractString, response)::SubString{String} re = access_threaded(status_line_regex_f, status_line_regex) if !exec(re, bytes) throw(ParseError(:INVALID_STATUS_LINE, bytes)) end response.version = HTTPVersion(group(1, re, bytes)) response.status = parse(Int, group(2, re, bytes)) return nextbytes(re, bytes) end """ Parse HTTP header-field. Return `Pair(field-name => field-value)` and a `SubString` containing the remaining header-field lines. """ function parse_header_field(bytes::SubString{String})::Tuple{Header,SubString{String}} # https://github.com/JuliaWeb/HTTP.jl/issues/796 # there may be certain scenarios where non-ascii characters are # included (illegally) in the headers; curl warns on these # "malformed headers" and ignores them. we attempt to re-encode # these from latin-1 => utf-8 and then try to parse. if !isvalid(bytes) @warn "malformed HTTP header detected; attempting to re-encode from Latin-1 to UTF8" bytes = SubString(iso8859_1_to_utf8(codeunits(bytes))) end # First look for: field-name ":" field-value re = access_threaded(header_field_regex_f, header_field_regex) if exec(re, bytes) return (group(1, re, bytes) => group(2, re, bytes)), nextbytes(re, bytes) end # Then check for empty termination line: re = access_threaded(empty_header_field_regex_f, empty_header_field_regex) if exec(re, bytes) return emptyheader, nextbytes(re, bytes) end # Finally look for obsolete line folding format: re = access_threaded(obs_fold_header_field_regex_f, obs_fold_header_field_regex) if exec(re, bytes) unfold = SubString(replace(group(2, re, bytes), r"\r?\n"=>"")) return (group(1, re, bytes) => unfold), nextbytes(re, bytes) end @label error throw(ParseError(:INVALID_HEADER_FIELD, bytes)) end # HTTP Chunked Transfer Coding """ Arbitrary limit to protect against denial of service attacks. """ const chunk_size_line_max = 64 const chunk_size_line_min = ncodeunits("0\r\n") @inline function skip_crlf(bytes, i=1) if @inbounds bytes[i] == UInt('\r') i += 1 end if @inbounds bytes[i] == UInt('\n') i += 1 end return i end """ Find `\\n` after chunk size in `bytes`. """ function find_end_of_chunk_size(bytes::AbstractVector{UInt8}) l = length(bytes) if l < chunk_size_line_min return 0 end if l > chunk_size_line_max l = chunk_size_line_max end i = skip_crlf(bytes) while i <= l if @inbounds bytes[i] == UInt('\n') return i end i += 1 end return 0 end """ find_end_of_trailer(bytes) -> length or 0 Find length of trailer delimited by `\\r\\n\\r\\n` (or starting with `\\r\\n`). [RFC7230 4.1](https://tools.ietf.org/html/rfc7230#section-4.1) """ find_end_of_trailer(bytes::AbstractVector{UInt8}) = length(bytes) < 2 ? 0 : bytes[2] == UInt8('\n') ? 2 : find_end_of_header(bytes) """ Arbitrary limit to protect against denial of service attacks. """ const chunk_size_limit = typemax(Int32) """ Parse HTTP chunk-size. Return number of bytes of chunk-data. chunk-size = 1*HEXDIG [RFC7230 4.1](https://tools.ietf.org/html/rfc7230#section-4.1) """ function parse_chunk_size(bytes::AbstractVector{UInt8})::Int chunk_size = Int64(0) i = skip_crlf(bytes) while true x = Int64(unhex[@inbounds bytes[i]]) if x == -1 break end chunk_size = chunk_size * Int64(16) + x if chunk_size > chunk_size_limit throw(ParseError(:CHUNK_SIZE_EXCEEDS_LIMIT, bytes)) end i += 1 end if i > 1 return Int(chunk_size) end throw(ParseError(:INVALID_CHUNK_SIZE, bytes)) end const unhex = Int8[ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1 ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1 ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1 ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 ] function __init__() # FIXME Consider turing off `PCRE.UTF` in `Regex.compile_options` # https://github.com/JuliaLang/julia/pull/26731#issuecomment-380676770 nt = @static if isdefined(Base.Threads, :maxthreadid) Threads.maxthreadid() else Threads.nthreads() end resize!(empty!(status_line_regex), nt) resize!(empty!(request_line_regex), nt) resize!(empty!(header_field_regex), nt) resize!(empty!(obs_fold_header_field_regex), nt) resize!(empty!(empty_header_field_regex), nt) return end end # module Parsers 0/opt/julia/packages/HTTP/sJD5V/src/parseutils.jlg# we define our own RegexAndMatchData, similar to the definition in Base # but we call create_match_data once in __init__ mutable struct RegexAndMatchData re::Regex match_data::Ptr{Cvoid} RegexAndMatchData(re::Regex) = new(re) # must create_match_data in __init__ end function initialize!(re::RegexAndMatchData) re.match_data = Base.PCRE.create_match_data(re.re.regex) return end @static if isdefined(Base, :RegexAndMatchData) """ Execute a regular expression without the overhead of `Base.Regex` """ exec(re::RegexAndMatchData, bytes, offset::Int=1) = Base.PCRE.exec(re.re.regex, bytes, offset-1, re.re.match_options, re.match_data) """ `SubString` containing the bytes following the matched regular expression. """ nextbytes(re::RegexAndMatchData, bytes) = SubString(bytes, unsafe_load(Base.PCRE.ovec_ptr(re.match_data), 2) + 1) """ `SubString` containing a regular expression match group. """ function group(i, re::RegexAndMatchData, bytes) p = Base.PCRE.ovec_ptr(re.match_data) SubString(bytes, unsafe_load(p, 2i+1) + 1, prevind(bytes, unsafe_load(p, 2i+2) + 1)) end function group(i, re::RegexAndMatchData, bytes, default) p = Base.PCRE.ovec_ptr(re.match_data) return unsafe_load(p, 2i+1) == Base.PCRE.UNSET ? default : SubString(bytes, unsafe_load(p, 2i+1) + 1, prevind(bytes, unsafe_load(p, 2i+2) + 1)) end else # old Regex style exec(re::RegexAndMatchData, bytes, offset::Int=1) = Base.PCRE.exec(re.re.regex, bytes, offset-1, re.re.match_options, re.re.match_data) nextbytes(re::RegexAndMatchData, bytes) = SubString(bytes, re.re.ovec[2]+1) group(i, re::RegexAndMatchData, bytes) = SubString(bytes, re.re.ovec[2i+1]+1, prevind(bytes, re.re.ovec[2i+2]+1)) group(i, re::RegexAndMatchData, bytes, default) = re.re.ovec[2i+1] == Base.PCRE.UNSET ? default : SubString(bytes, re.re.ovec[2i+1]+1, prevind(bytes, re.re.ovec[2i+2]+1)) end 1/opt/julia/packages/HTTP/sJD5V/src/Connections.jlX""" This module provides the [`newconnection`](@ref) function with support for: - Opening TCP and SSL connections. - Reusing connections for multiple Request/Response Messages This module defines a [`Connection`](@ref) struct to manage the lifetime of a connection and its reuse. Methods are provided for `eof`, `readavailable`, `unsafe_write` and `close`. This allows the `Connection` object to act as a proxy for the `TCPSocket` or `SSLContext` that it wraps. [`POOLS`](@ref) are used to manage connection pooling. Connections are identified by their host, port, whether they require ssl verification, and whether they are a client or server connection. If a subsequent request matches these properties of a previous connection and limits are respected (reuse limit, idle timeout), and it wasn't otherwise remotely closed, a connection will be reused. """ module Connections export Connection, newconnection, releaseconnection, getrawstream, inactiveseconds, shouldtimeout, default_connection_limit, set_default_connection_limit!, Pool using Sockets, LoggingExtras, NetworkOptions using MbedTLS: SSLConfig, SSLContext, setup!, associate!, hostname!, handshake! using MbedTLS, OpenSSL, ConcurrentUtilities using ..IOExtras, ..Conditions, ..Exceptions const nolimit = typemax(Int) taskid(t=current_task()) = string(hash(t) & 0xffff, base=16, pad=4) const default_connection_limit = Ref{Int}() function __init__() # default connection limit is 4x the number of threads # this was chosen after some empircal benchmarking on aws/azure machines # where, for non-data-intensive workloads, having at least 4x ensured # there was no artificial restriction on overall throughput default_connection_limit[] = max(16, Threads.nthreads() * 4) nosslcontext[] = OpenSSL.SSLContext(OpenSSL.TLSClientMethod()) TCP_POOL[] = CPool{Sockets.TCPSocket}(default_connection_limit[]) MBEDTLS_POOL[] = CPool{MbedTLS.SSLContext}(default_connection_limit[]) OPENSSL_POOL[] = CPool{OpenSSL.SSLStream}(default_connection_limit[]) return end function set_default_connection_limit!(n) default_connection_limit[] = n # reinitialize the global connection pools TCP_POOL[] = CPool{Sockets.TCPSocket}(n) MBEDTLS_POOL[] = CPool{MbedTLS.SSLContext}(n) OPENSSL_POOL[] = CPool{OpenSSL.SSLStream}(n) return end """ Connection A `Sockets.TCPSocket`, `MbedTLS.SSLContext` or `OpenSSL.SSLStream` connection to a HTTP `host` and `port`. Fields: - `host::String` - `port::String`, exactly as specified in the URI (i.e. may be empty). - `idle_timeout`, No. of seconds to maintain connection after last request/response. - `require_ssl_verification`, whether ssl verification is required for an ssl connection - `keepalive`, whether the tcp socket should have keepalive enabled - `peerip`, remote IP adress (used for debug/log messages). - `peerport`, remote TCP port number (used for debug/log messages). - `localport`, local TCP port number (used for debug messages). - `io::T`, the `Sockets.TCPSocket`, `MbedTLS.SSLContext` or `OpenSSL.SSLStream`. - `clientconnection::Bool`, whether the Connection was created from client code (as opposed to server code) - `buffer::IOBuffer`, left over bytes read from the connection after the end of a response header (or chunksize). These bytes are usually part of the response body. - `timestamp`, time data was last received. - `readable`, whether the Connection object is readable - `writable`, whether the Connection object is writable """ mutable struct Connection{IO_t <: IO} <: IO host::String port::String idle_timeout::Int require_ssl_verification::Bool keepalive::Bool peerip::IPAddr # for debugging/logging peerport::UInt16 # for debugging/logging localport::UInt16 # debug only io::IO_t clientconnection::Bool buffer::IOBuffer timestamp::Float64 readable::Bool writable::Bool writebuffer::IOBuffer state::Any # populated & used by Servers code end has_tcpsocket(c::Connection) = applicable(tcpsocket, c.io) IOExtras.tcpsocket(c::Connection) = tcpsocket(c.io) """ connectionkey Used for "hashing" a Connection object on just the key properties necessary for determining connection re-useability. That is, when a new request calls `newconnection`, we take the request parameters of host and port, and if ssl verification is required, if keepalive is enabled, and if an existing Connection was already created with the exact. same parameters, we can re-use it (as long as it's not already being used, obviously). """ connectionkey(x::Connection) = (x.host, x.port, x.require_ssl_verification, x.keepalive, x.clientconnection) const ConnectionKeyType = Tuple{AbstractString, AbstractString, Bool, Bool, Bool} Connection(host::AbstractString, port::AbstractString, idle_timeout::Int, require_ssl_verification::Bool, keepalive::Bool, io::T, client=true) where {T}= Connection{T}(host, port, idle_timeout, require_ssl_verification, keepalive, safe_getpeername(io)..., localport(io), io, client, PipeBuffer(), time(), false, false, IOBuffer(), nothing) Connection(io; require_ssl_verification::Bool=true, keepalive::Bool=true) = Connection("", "", 0, require_ssl_verification, keepalive, io, false) getrawstream(c::Connection) = c.io inactiveseconds(c::Connection)::Float64 = time() - c.timestamp shouldtimeout(c::Connection, readtimeout) = !isreadable(c) || inactiveseconds(c) > readtimeout Base.unsafe_write(c::Connection, p::Ptr{UInt8}, n::UInt) = unsafe_write(c.io, p, n) Base.isopen(c::Connection) = isopen(c.io) """ flush(c::Connection) Flush a TCP buffer by toggling the Nagle algorithm off and on again for a socket. This forces the socket to send whatever data is within its buffer immediately, rather than waiting 10's of milliseconds for the buffer to fill more. """ function Base.flush(c::Connection) # Flushing the TCP buffer requires support for `Sockets.nagle()` # which was only added in Julia v1.3 @static if VERSION >= v"1.3" sock = tcpsocket(c) # I don't understand why uninitializd sockets can get here, but they can if sock.status ∉ (Base.StatusInit, Base.StatusUninit) && isopen(sock) Sockets.nagle(sock, false) Sockets.nagle(sock, true) end end end Base.isreadable(c::Connection) = c.readable Base.iswritable(c::Connection) = c.writable function Base.eof(c::Connection) @require isreadable(c) || !isopen(c) if bytesavailable(c) > 0 return false end return eof(c.io) end Base.bytesavailable(c::Connection) = bytesavailable(c.buffer) + bytesavailable(c.io) function Base.read(c::Connection, nb::Int) nb = min(nb, bytesavailable(c)) bytes = Base.StringVector(nb) GC.@preserve bytes unsafe_read(c, pointer(bytes), nb) return bytes end function Base.read(c::Connection, ::Type{UInt8}) if bytesavailable(c.buffer) == 0 read_to_buffer(c) end return read(c.buffer, UInt8) end function Base.unsafe_read(c::Connection, p::Ptr{UInt8}, n::UInt) l = bytesavailable(c.buffer) if l > 0 nb = min(l, n) unsafe_read(c.buffer, p, nb) p += nb n -= nb c.timestamp = time() end if n > 0 # try-catch underlying errors here # as the Connection object, we don't really care # if the underlying socket was closed/terminated # or just plain reached EOF, so we catch any # Base.IOErrors and just throw as EOFError # that way we get more consistent errors thrown # at the headers/body parsing level try unsafe_read(c.io, p, n) c.timestamp = time() catch e e isa Base.IOError && throw(EOFError()) rethrow(e) end end return nothing end function read_to_buffer(c::Connection, sizehint=4096) buf = c.buffer # Reset the buffer if it is empty. if bytesavailable(buf) == 0 buf.size = 0 buf.ptr = 1 end # Wait for data. if eof(c.io) throw(EOFError()) end # Read from stream into buffer. n = min(sizehint, bytesavailable(c.io)) buf = c.buffer Base.ensureroom(buf, n) GC.@preserve buf unsafe_read(c.io, pointer(buf.data, buf.size + 1), n) buf.size += n end """ Read until `find_delimiter(bytes)` returns non-zero. Return view of bytes up to the delimiter. """ function IOExtras.readuntil(c::Connection, f::F #=Vector{UInt8} -> Int=#, sizehint=4096) where {F <: Function} buf = c.buffer if bytesavailable(buf) == 0 read_to_buffer(c, sizehint) end while isempty(begin bytes = IOExtras.readuntil(buf, f) end) read_to_buffer(c, sizehint) end return bytes end """ startwrite(::Connection) """ function IOExtras.startwrite(c::Connection) @require !iswritable(c) c.writable = true @debugv 3 "👁 Start write:$c" return end """ closewrite(::Connection) Signal that an entire Request Message has been written to the `Connection`. """ function IOExtras.closewrite(c::Connection) @require iswritable(c) c.writable = false @debugv 3 "🗣 Write done: $c" flush(c) return end """ startread(::Connection) """ function IOExtras.startread(c::Connection) @require !isreadable(c) c.timestamp = time() c.readable = true @debugv 3 "👁 Start read: $c" return end """ Wait for `c` to receive data or reach EOF. Close `c` on EOF. TODO: or if response data arrives when no request was sent (isreadable == false). """ function monitor_idle_connection(c::Connection) try if eof(c.io) ;@debugv 3 "💀 Closed: $c" close(c.io) end catch ex @try Base.IOError close(c.io) ex isa Base.IOError || rethrow() end nothing end """ closeread(::Connection) Signal that an entire Response Message has been read from the `Connection`. """ function IOExtras.closeread(c::Connection) @require isreadable(c) c.readable = false @debugv 3 "✉️ Read done: $c" if c.clientconnection t = Threads.@spawn monitor_idle_connection(c) @isdefined(errormonitor) && errormonitor(t) end return end Base.wait_close(c::Connection) = Base.wait_close(tcpsocket(c)) function Base.close(c::Connection) if iswritable(c) closewrite(c) end if isreadable(c) closeread(c) end try close(c.io) if bytesavailable(c) > 0 purge(c) end catch # ignore errors closing underlying socket end return end """ purge(::Connection) Remove unread data from a `Connection`. """ function purge(c::Connection) @require !isopen(c.io) while !eof(c.io) readavailable(c.io) end c.buffer.size = 0 c.buffer.ptr = 1 @ensure bytesavailable(c) == 0 end const CPool{T} = ConcurrentUtilities.Pool{ConnectionKeyType, Connection{T}} """ HTTP.Pool(max::Int=HTTP.default_connection_limit[]) Connection pool for managing the reuse of HTTP connections. `max` controls the maximum number of concurrent connections allowed and defaults to the `HTTP.default_connection_limit` value. A pool can be passed to any of the `HTTP.request` methods via the `pool` keyword argument. """ struct Pool lock::ReentrantLock tcp::CPool{Sockets.TCPSocket} mbedtls::CPool{MbedTLS.SSLContext} openssl::CPool{OpenSSL.SSLStream} other::IdDict{Type, CPool} max::Int end function Pool(max::Union{Int, Nothing}=nothing) max = something(max, default_connection_limit[]) return Pool(ReentrantLock(), CPool{Sockets.TCPSocket}(max), CPool{MbedTLS.SSLContext}(max), CPool{OpenSSL.SSLStream}(max), IdDict{Type, CPool}(), max, ) end # Default HTTP global connection pools const TCP_POOL = Ref{CPool{Sockets.TCPSocket}}() const MBEDTLS_POOL = Ref{CPool{MbedTLS.SSLContext}}() const OPENSSL_POOL = Ref{CPool{OpenSSL.SSLStream}}() const OTHER_POOL = Lockable(IdDict{Type, CPool}()) getpool(::Nothing, ::Type{Sockets.TCPSocket}) = TCP_POOL[] getpool(::Nothing, ::Type{MbedTLS.SSLContext}) = MBEDTLS_POOL[] getpool(::Nothing, ::Type{OpenSSL.SSLStream}) = OPENSSL_POOL[] getpool(::Nothing, ::Type{T}) where {T} = Base.@lock OTHER_POOL get!(OTHER_POOL[], T) do CPool{T}(default_connection_limit[]) end function getpool(pool::Pool, ::Type{T})::CPool{T} where {T} if T === Sockets.TCPSocket return pool.tcp elseif T === MbedTLS.SSLContext return pool.mbedtls elseif T === OpenSSL.SSLStream return pool.openssl else return Base.@lock pool.lock get!(() -> CPool{T}(pool.max), pool.other, T) end end """ closeall(pool::HTTP.Pool=nothing) Remove and close all connections in the `pool` to avoid any connection reuse. If `pool` is not specified, the default global pools are closed. """ function closeall(pool::Union{Nothing, Pool}=nothing) if pool === nothing drain!(TCP_POOL[]) drain!(MBEDTLS_POOL[]) drain!(OPENSSL_POOL[]) Base.@lock OTHER_POOL foreach(drain!, values(OTHER_POOL[])) else drain!(pool.tcp) drain!(pool.mbedtls) drain!(pool.openssl) Base.@lock pool.lock foreach(drain!, values(pool.other)) end return end function connection_isvalid(c, idle_timeout) check = isopen(c) && inactiveseconds(c) <= idle_timeout check || close(c) return check end @noinline connection_limit_warning(cl) = cl === nothing || @warn "connection_limit no longer supported as a keyword argument; use `HTTP.set_default_connection_limit!($cl)` before any requests are made or construct a shared pool via `POOL = HTTP.Pool($cl)` and pass to each request like `pool=POOL` instead." const DEFAULT_CONNECT_TIMEOUT = Ref{Int}(30) """ newconnection(type, host, port) -> Connection Find a reusable `Connection` in the `pool`, or create a new `Connection` if required. """ function newconnection(::Type{T}, host::AbstractString, port::AbstractString; pool::Union{Nothing, Pool}=nothing, connection_limit=nothing, forcenew::Bool=false, idle_timeout=typemax(Int), connect_timeout::Int=DEFAULT_CONNECT_TIMEOUT[], require_ssl_verification::Bool=NetworkOptions.verify_host(host, "SSL"), keepalive::Bool=true, kw...) where {T <: IO} connection_limit_warning(connection_limit) return acquire( getpool(pool, T), (host, port, require_ssl_verification, keepalive, true); forcenew=forcenew, isvalid=c->connection_isvalid(c, Int(idle_timeout))) do Connection(host, port, idle_timeout, require_ssl_verification, keepalive, connect_timeout > 0 ? try_with_timeout(_ -> getconnection(T, host, port; require_ssl_verification=require_ssl_verification, keepalive=keepalive, kw...), connect_timeout) : getconnection(T, host, port; require_ssl_verification=require_ssl_verification, keepalive=keepalive, kw...) ) end end function releaseconnection(c::Connection{T}, reuse; pool::Union{Nothing, Pool}=nothing, kw...) where {T} c.timestamp = time() release(getpool(pool, T), connectionkey(c), reuse ? c : nothing) end function keepalive!(tcp) Base.iolock_begin() try Base.check_open(tcp) msg = ccall(:uv_tcp_keepalive, Cint, (Ptr{Nothing}, Cint, Cuint), tcp.handle, 1, 1) Base.uv_error("failed to set keepalive on tcp socket", msg) finally Base.iolock_end() end return end struct ConnectTimeout <: Exception host port end function checkconnected(tcp) if tcp.status == Base.StatusConnecting close(tcp) return false end return true end function getconnection(::Type{TCPSocket}, host::AbstractString, port::AbstractString; # set keepalive to true by default since it's cheap and helps keep long-running requests/responses # alive in the face of heavy workloads where Julia's task scheduler might take a while to # keep up with midflight requests keepalive::Bool=true, readtimeout::Int=0, kw...)::TCPSocket p::UInt = isempty(port) ? UInt(80) : parse(UInt, port) @debugv 2 "TCP connect: $host:$p..." addrs = Sockets.getalladdrinfo(host) err = ErrorException("failed to connect") for addr in addrs try tcp = Sockets.connect(addr, p) keepalive && keepalive!(tcp) return tcp catch e err = e end end throw(err) end const nosslconfig = SSLConfig() const nosslcontext = Ref{OpenSSL.SSLContext}() const default_sslconfig = Ref{Union{Nothing, SSLConfig}}(nothing) const noverify_sslconfig = Ref{Union{Nothing, SSLConfig}}(nothing) function global_sslconfig(require_ssl_verification::Bool)::SSLConfig if default_sslconfig[] === nothing default_sslconfig[] = SSLConfig(true) noverify_sslconfig[] = SSLConfig(false) end if haskey(ENV, "HTTP_CA_BUNDLE") MbedTLS.ca_chain!(default_sslconfig[], MbedTLS.crt_parse(read(ENV["HTTP_CA_BUNDLE"], String))) elseif haskey(ENV, "CURL_CA_BUNDLE") MbedTLS.ca_chain!(default_sslconfig[], MbedTLS.crt_parse(read(ENV["CURL_CA_BUNDLE"], String))) end return require_ssl_verification ? default_sslconfig[] : noverify_sslconfig[] end function global_sslcontext()::OpenSSL.SSLContext @static if isdefined(OpenSSL, :ca_chain!) if haskey(ENV, "HTTP_CA_BUNDLE") sslcontext = OpenSSL.SSLContext(OpenSSL.TLSClientMethod()) OpenSSL.ca_chain!(sslcontext, ENV["HTTP_CA_BUNDLE"]) return sslcontext elseif haskey(ENV, "CURL_CA_BUNDLE") sslcontext = OpenSSL.SSLContext(OpenSSL.TLSClientMethod()) OpenSSL.ca_chain!(sslcontext, ENV["CURL_CA_BUNDLE"]) return sslcontext end end return nosslcontext[] end function getconnection(::Type{SSLContext}, host::AbstractString, port::AbstractString; kw...)::SSLContext port = isempty(port) ? "443" : port @debugv 2 "SSL connect: $host:$port..." tcp = getconnection(TCPSocket, host, port; kw...) return sslconnection(SSLContext, tcp, host; kw...) end function getconnection(::Type{SSLStream}, host::AbstractString, port::AbstractString; kw...)::SSLStream port = isempty(port) ? "443" : port @debugv 2 "SSL connect: $host:$port..." tcp = getconnection(TCPSocket, host, port; kw...) return sslconnection(SSLStream, tcp, host; kw...) end function sslconnection(::Type{SSLStream}, tcp::TCPSocket, host::AbstractString; require_ssl_verification::Bool=NetworkOptions.verify_host(host, "SSL"), sslconfig::OpenSSL.SSLContext=nosslcontext[], kw...)::SSLStream if sslconfig === nosslcontext[] sslconfig = global_sslcontext() end # Create SSL stream. ssl_stream = SSLStream(sslconfig, tcp) OpenSSL.hostname!(ssl_stream, host) OpenSSL.connect(ssl_stream; require_ssl_verification) return ssl_stream end function sslconnection(::Type{SSLContext}, tcp::TCPSocket, host::AbstractString; require_ssl_verification::Bool=NetworkOptions.verify_host(host, "SSL"), sslconfig::SSLConfig=nosslconfig, kw...)::SSLContext if sslconfig === nosslconfig sslconfig = global_sslconfig(require_ssl_verification) end io = SSLContext() setup!(io, sslconfig) associate!(io, tcp) hostname!(io, host) handshake!(io) return io end function sslupgrade(::Type{IOType}, c::Connection{T}, host::AbstractString; pool::Union{Nothing, Pool}=nothing, require_ssl_verification::Bool=NetworkOptions.verify_host(host, "SSL"), keepalive::Bool=true, readtimeout::Int=0, kw...)::Connection{IOType} where {T, IOType} # initiate the upgrade to SSL # if the upgrade fails, an error will be thrown and the original c will be closed # in ConnectionRequest tls = if readtimeout > 0 try_with_timeout(readtimeout) do _ sslconnection(IOType, c.io, host; require_ssl_verification=require_ssl_verification, keepalive=keepalive, kw...) end else sslconnection(IOType, c.io, host; require_ssl_verification=require_ssl_verification, keepalive=keepalive, kw...) end # success, now we turn it into a new Connection conn = Connection(host, "", 0, require_ssl_verification, keepalive, tls) # release the "old" one, but don't return the connection since we're hijacking the socket release(getpool(pool, T), connectionkey(c)) # and return the new one return acquire(() -> conn, getpool(pool, IOType), connectionkey(conn); forcenew=true) end function Base.show(io::IO, c::Connection) nwaiting = has_tcpsocket(c) ? bytesavailable(tcpsocket(c)) : 0 print( io, tcpstatus(c), " ", "$(lpad(round(Int, time() - c.timestamp), 3))s ", c.host, ":", c.port != "" ? c.port : Int(c.peerport), ":", Int(c.localport), bytesavailable(c.buffer) > 0 ? " $(bytesavailable(c.buffer))-byte excess" : "", nwaiting > 0 ? " $nwaiting bytes waiting" : "", has_tcpsocket(c) ? " $(Base._fd(tcpsocket(c)))" : "") end function tcpstatus(c::Connection) if !has_tcpsocket(c) return "" end s = Base.uv_status_string(tcpsocket(c)) if s == "connecting" return "🔜🔗" elseif s == "open" return "🔗 " elseif s == "active" return "🔁 " elseif s == "paused" return "⏸ " elseif s == "closing" return "🔜💀" elseif s == "closed" return "💀 " else return s end end end # module Connections 1/opt/julia/packages/HTTP/sJD5V/src/StatusCodes.jl""" This module provides HTTP status code constatnts and related functions """ module StatusCodes export statustext # Status code definitions const CONTINUE = 100 const SWITCHING_PROTOCOLS = 101 const PROCESSING = 102 const EARLY_HINTS = 103 const OK = 200 const CREATED = 201 const ACCEPTED = 202 const NON_AUTHORITATIVE_INFORMATION = 203 const NO_CONTENT = 204 const RESET_CONTENT = 205 const PARTIAL_CONTENT = 206 const MULTI_STATUS = 207 const ALREADY_REPORTED = 208 const IM_USED = 226 const MULTIPLE_CHOICES = 300 const MOVED_PERMANENTLY = 301 const MOVED_TEMPORARILY = 302 const SEE_OTHER = 303 const NOT_MODIFIED = 304 const USE_PROXY = 305 const TEMPORARY_REDIRECT = 307 const PERMANENT_REDIRECT = 308 const BAD_REQUEST = 400 const UNAUTHORIZED = 401 const PAYMENT_REQUIRED = 402 const FORBIDDEN = 403 const NOT_FOUND = 404 const METHOD_NOT_ALLOWED = 405 const NOT_ACCEPTABLE = 406 const PROXY_AUTHENTICATION_REQUIRED = 407 const REQUEST_TIME_OUT = 408 const CONFLICT = 409 const GONE = 410 const LENGTH_REQUIRED = 411 const PRECONDITION_FAILED = 412 const REQUEST_ENTITY_TOO_LARGE = 413 const REQUEST_URI_TOO_LARGE = 414 const UNSUPPORTED_MEDIA_TYPE = 415 const REQUESTED_RANGE_NOT_SATISFIABLE = 416 const EXPECTATION_FAILED = 417 const IM_A_TEAPOT = 418 const MISDIRECTED_REQUEST = 421 const UNPROCESSABLE_ENTITY = 422 const LOCKED = 423 const FAILED_DEPENDENCY = 424 const UNORDERED_COLLECTION = 425 const UPGRADE_REQUIRED = 426 const PRECONDITION_REQUIRED = 428 const TOO_MANY_REQUESTS = 429 const REQUEST_HEADER_FIELDS_TOO_LARGE = 431 const LOGIN_TIMEOUT = 440 const NGINX_ERROR_NO_RESPONSE = 444 const UNAVAILABLE_FOR_LEGAL_REASONS = 451 const NGINX_ERROR_SSL_CERTIFICATE_ERROR = 495 const NGINX_ERROR_SSL_CERTIFICATE_REQUIRED = 496 const NGINX_ERROR_HTTP_TO_HTTPS = 497 const NGINX_ERROR_OR_ANTIVIRUS_INTERCEPTED_REQUEST_OR_ARCGIS_ERROR = 499 const INTERNAL_SERVER_ERROR = 500 const NOT_IMPLEMENTED = 501 const BAD_GATEWAY = 502 const SERVICE_UNAVAILABLE = 503 const GATEWAY_TIME_OUT = 504 const HTTP_VERSION_NOT_SUPPORTED = 505 const VARIANT_ALSO_NEGOTIATES = 506 const INSUFFICIENT_STORAGE = 507 const LOOP_DETECTED = 508 const BANDWIDTH_LIMIT_EXCEEDED = 509 const NOT_EXTENDED = 510 const NETWORK_AUTHENTICATION_REQUIRED = 511 const CLOUDFLARE_SERVER_ERROR_UNKNOWN = 520 const CLOUDFLARE_SERVER_ERROR_CONNECTION_REFUSED = 521 const CLOUDFLARE_SERVER_ERROR_CONNECTION_TIMEOUT = 522 const CLOUDFLARE_SERVER_ERROR_ORIGIN_SERVER_UNREACHABLE = 523 const CLOUDFLARE_SERVER_ERROR_A_TIMEOUT = 524 const CLOUDFLARE_SERVER_ERROR_CONNECTION_FAILED = 525 const CLOUDFLARE_SERVER_ERROR_INVALID_SSL_CERITIFICATE = 526 const CLOUDFLARE_SERVER_ERROR_RAILGUN_ERROR = 527 const SITE_FROZEN = 530 include("status_messages.jl") end # module StatusCodes 5/opt/julia/packages/HTTP/sJD5V/src/status_messages.jl""" statustext(::Int) -> String `String` representation of a HTTP status code. ## Examples ```julia julia> statustext(200) "OK" julia> statustext(404) "Not Found" ``` """ statustext(status) = Base.get(STATUS_MESSAGES, status, "Unknown Code") const STATUS_MESSAGES = (()->begin v = fill("Unknown Code", 530) v[CONTINUE] = "Continue" v[SWITCHING_PROTOCOLS] = "Switching Protocols" v[PROCESSING] = "Processing" # RFC 2518 => obsoleted by RFC 4918 v[EARLY_HINTS] = "Early Hints" v[OK] = "OK" v[CREATED] = "Created" v[ACCEPTED] = "Accepted" v[NON_AUTHORITATIVE_INFORMATION] = "Non-Authoritative Information" v[NO_CONTENT] = "No Content" v[RESET_CONTENT] = "Reset Content" v[PARTIAL_CONTENT] = "Partial Content" v[MULTI_STATUS] = "Multi-Status" # RFC4918 v[ALREADY_REPORTED] = "Already Reported" # RFC5842 v[IM_USED] = "IM Used" # RFC3229 v[MULTIPLE_CHOICES] = "Multiple Choices" v[MOVED_PERMANENTLY] = "Moved Permanently" v[MOVED_TEMPORARILY] = "Moved Temporarily" v[SEE_OTHER] = "See Other" v[NOT_MODIFIED] = "Not Modified" v[USE_PROXY] = "Use Proxy" v[TEMPORARY_REDIRECT] = "Temporary Redirect" v[PERMANENT_REDIRECT] = "Permanent Redirect" # RFC7238 v[BAD_REQUEST] = "Bad Request" v[UNAUTHORIZED] = "Unauthorized" v[PAYMENT_REQUIRED] = "Payment Required" v[FORBIDDEN] = "Forbidden" v[NOT_FOUND] = "Not Found" v[METHOD_NOT_ALLOWED] = "Method Not Allowed" v[NOT_ACCEPTABLE] = "Not Acceptable" v[PROXY_AUTHENTICATION_REQUIRED] = "Proxy Authentication Required" v[REQUEST_TIME_OUT] = "Request Time-out" v[CONFLICT] = "Conflict" v[GONE] = "Gone" v[LENGTH_REQUIRED] = "Length Required" v[PRECONDITION_FAILED] = "Precondition Failed" v[REQUEST_ENTITY_TOO_LARGE] = "Request Entity Too Large" v[REQUEST_URI_TOO_LARGE] = "Request-URI Too Large" v[UNSUPPORTED_MEDIA_TYPE] = "Unsupported Media Type" v[REQUESTED_RANGE_NOT_SATISFIABLE] = "Requested Range Not Satisfiable" v[EXPECTATION_FAILED] = "Expectation Failed" v[IM_A_TEAPOT] = "I'm a teapot" # RFC 2324 v[MISDIRECTED_REQUEST] = "Misdirected Request" # RFC 7540 v[UNPROCESSABLE_ENTITY] = "Unprocessable Entity" # RFC 4918 v[LOCKED] = "Locked" # RFC 4918 v[FAILED_DEPENDENCY] = "Failed Dependency" # RFC 4918 v[UNORDERED_COLLECTION] = "Unordered Collection" # RFC 4918 v[UPGRADE_REQUIRED] = "Upgrade Required" # RFC 2817 v[PRECONDITION_REQUIRED] = "Precondition Required" # RFC 6585 v[TOO_MANY_REQUESTS] = "Too Many Requests" # RFC 6585 v[REQUEST_HEADER_FIELDS_TOO_LARGE] = "Request Header Fields Too Large" # RFC 6585 v[LOGIN_TIMEOUT] = "Login Timeout" v[NGINX_ERROR_NO_RESPONSE] = "nginx error: No Response" v[UNAVAILABLE_FOR_LEGAL_REASONS] = "Unavailable For Legal Reasons" # RFC7725 v[NGINX_ERROR_SSL_CERTIFICATE_ERROR] = "nginx error: SSL Certificate Error" v[NGINX_ERROR_SSL_CERTIFICATE_REQUIRED] = "nginx error: SSL Certificate Required" v[NGINX_ERROR_HTTP_TO_HTTPS] = "nginx error: HTTP -> HTTPS" v[NGINX_ERROR_OR_ANTIVIRUS_INTERCEPTED_REQUEST_OR_ARCGIS_ERROR] = "nginx error or Antivirus intercepted request or ArcGIS error" v[INTERNAL_SERVER_ERROR] = "Internal Server Error" v[NOT_IMPLEMENTED] = "Not Implemented" v[BAD_GATEWAY] = "Bad Gateway" v[SERVICE_UNAVAILABLE] = "Service Unavailable" v[GATEWAY_TIME_OUT] = "Gateway Time-out" v[HTTP_VERSION_NOT_SUPPORTED] = "HTTP Version Not Supported" v[VARIANT_ALSO_NEGOTIATES] = "Variant Also Negotiates" # RFC 2295 v[INSUFFICIENT_STORAGE] = "Insufficient Storage" # RFC 4918 v[LOOP_DETECTED] = "Loop Detected" # RFC5842 v[BANDWIDTH_LIMIT_EXCEEDED] = "Bandwidth Limit Exceeded" v[NOT_EXTENDED] = "Not Extended" # RFC 2774 v[NETWORK_AUTHENTICATION_REQUIRED] = "Network Authentication Required" # RFC 6585 v[CLOUDFLARE_SERVER_ERROR_UNKNOWN] = "CloudFlare Server Error: Unknown" v[CLOUDFLARE_SERVER_ERROR_CONNECTION_REFUSED] = "CloudFlare Server Error: Connection Refused" v[CLOUDFLARE_SERVER_ERROR_CONNECTION_TIMEOUT] = "CloudFlare Server Error: Connection Timeout" v[CLOUDFLARE_SERVER_ERROR_ORIGIN_SERVER_UNREACHABLE] = "CloudFlare Server Error: Origin Server Unreachable" v[CLOUDFLARE_SERVER_ERROR_A_TIMEOUT] = "CloudFlare Server Error: A Timeout" v[CLOUDFLARE_SERVER_ERROR_CONNECTION_FAILED] = "CloudFlare Server Error: Connection Failed" v[CLOUDFLARE_SERVER_ERROR_INVALID_SSL_CERITIFICATE] = "CloudFlare Server Error: Invalid SSL Ceritificate" v[CLOUDFLARE_SERVER_ERROR_RAILGUN_ERROR] = "CloudFlare Server Error: Railgun Error" v[SITE_FROZEN] = "Site Frozen" return v end)() ./opt/julia/packages/HTTP/sJD5V/src/Messages.jlQ""" The `Messages` module defines structs that represent [`HTTP.Request`](@ref) and [`HTTP.Response`](@ref) Messages. The `Response` struct has a `request` field that points to the corresponding `Request`; and the `Request` struct has a `response` field. The `Request` struct also has a `parent` field that points to a `Response` in the case of HTTP redirects that occur and are followed. The Messages module defines `IO` `read` and `write` methods for Messages but it does not deal with URIs, creating connections, or executing requests. The `read` methods throw `EOFError` exceptions if input data is incomplete. and call parser functions that may throw `HTTP.ParsingError` exceptions. The `read` and `write` methods may also result in low level `IO` exceptions. ### Sending Messages Messages are formatted and written to an `IO` stream by [`Base.write(::IO,::HTTP.Messages.Message)`](@ref) and/or [`HTTP.Messages.writeheaders`](@ref). ### Receiving Messages Messages are parsed from `IO` stream data by [`HTTP.Messages.readheaders`](@ref). This function calls [`HTTP.Parsers.parse_header_field`](@ref) and passes each header-field to [`HTTP.Messages.appendheader`](@ref). ### Headers Headers are represented by `Vector{Pair{String,String}}`. As compared to `Dict{String,String}` this allows [repeated header fields and preservation of order](https://tools.ietf.org/html/rfc7230#section-3.2.2). Header values can be accessed by name using [`HTTP.header`](@ref) and [`HTTP.setheader`](@ref) (case-insensitive). The [`HTTP.appendheader`](@ref) function handles combining multi-line values, repeated header fields and special handling of multiple `Set-Cookie` headers. ### Bodies The `HTTP.Message` structs represent the message body by default as `Vector{UInt8}`. If `IO` or iterator objects are passed as the body, they will be stored as is in the `Request`/`Response` `body` field. """ module Messages export Message, Request, Response, reset!, status, method, headers, uri, body, resource, iserror, isredirect, retryablebody, retryable, retrylimitreached, ischunked, issafe, isidempotent, header, hasheader, headercontains, setheader, defaultheader!, appendheader, removeheader, mkheaders, readheaders, headerscomplete, readchunksize, writeheaders, writestartline, bodylength, unknown_length, payload, decode, sprintcompact using URIs, CodecZlib using ..Pairs, ..IOExtras, ..Parsers, ..Strings, ..Forms, ..Conditions using ..Connections, ..StatusCodes const nobody = UInt8[] const unknown_length = typemax(Int) sprintcompact(x) = sprint(show, x; context=:compact => true) abstract type Message end # HTTP Response """ HTTP.Response(status, headers::HTTP.Headers, body; request=nothing) HTTP.Response(status, body) HTTP.Response(body) Represents an HTTP response message with fields: - `version::HTTPVersion` [RFC7230 2.6](https://tools.ietf.org/html/rfc7230#section-2.6) - `status::Int16` [RFC7230 3.1.2](https://tools.ietf.org/html/rfc7230#section-3.1.2) [RFC7231 6](https://tools.ietf.org/html/rfc7231#section-6) - `headers::Vector{Pair{String,String}}` [RFC7230 3.2](https://tools.ietf.org/html/rfc7230#section-3.2) - `body::Vector{UInt8}` or `body::IO` [RFC7230 3.3](https://tools.ietf.org/html/rfc7230#section-3.3) - `request`, the `Request` that yielded this `Response`. """ mutable struct Response <: Message version::HTTPVersion status::Int16 headers::Headers body::Any # Usually Vector{UInt8} or IO request::Union{Message, Nothing} # Union{Request, Nothing} end function Response(status::Integer, headers, body; version=HTTPVersion(1, 1), request=nothing) b = isbytes(body) ? bytes(body) : something(body, nobody) @assert (request isa Request || request === nothing) return Response(version, status, mkheaders(headers), b, request) end # legacy constructor Response(status::Integer, headers=[]; body=nobody, request=nothing) = Response(status, headers, body; request) Response() = Request().response Response(s::Int, body::AbstractVector{UInt8}) = Response(s; body=body) Response(s::Int, body::AbstractString) = Response(s; body=bytes(body)) Response(body) = Response(200; body=body) Base.convert(::Type{Response}, s::AbstractString) = Response(s) function reset!(r::Response) r.version = HTTPVersion(1, 1) r.status = 0 if !isempty(r.headers) empty!(r.headers) end delete!(r.request.context, :response_body) return end status(r::Response) = getfield(r, :status) headers(r::Response) = getfield(r, :headers) body(r::Response) = getfield(r, :body) # HTTP Request const Context = Dict{Symbol, Any} """ HTTP.Request( method, target, headers=[], body=nobody; version=v"1.1", url::URI=URI(), responsebody=nothing, parent=nothing, context=HTTP.Context() ) Represents a HTTP Request Message with fields: - `method::String` [RFC7230 3.1.1](https://tools.ietf.org/html/rfc7230#section-3.1.1) - `target::String` [RFC7230 5.3](https://tools.ietf.org/html/rfc7230#section-5.3) - `version::HTTPVersion` [RFC7230 2.6](https://tools.ietf.org/html/rfc7230#section-2.6) - `headers::HTTP.Headers` [RFC7230 3.2](https://tools.ietf.org/html/rfc7230#section-3.2) - `body::Union{Vector{UInt8}, IO}` [RFC7230 3.3](https://tools.ietf.org/html/rfc7230#section-3.3) - `response`, the `Response` to this `Request` - `url::URI`, the full URI of the request - `parent`, the `Response` (if any) that led to this request (e.g. in the case of a redirect). [RFC7230 6.4](https://tools.ietf.org/html/rfc7231#section-6.4) - `context`, a `Dict{Symbol, Any}` store used by middleware to share state """ mutable struct Request <: Message method::String target::String version::HTTPVersion headers::Headers body::Any # Usually Vector{UInt8} or some kind of IO response::Response url::URI parent::Union{Response, Nothing} context::Context end Request() = Request("", "") function Request( method::String, target, headers=[], body=nobody; version=HTTPVersion(1, 1), url::URI=URI(), responsebody=nothing, parent=nothing, context=Context() ) b = isbytes(body) ? bytes(body) : body r = Request(method, target == "" ? "/" : target, version, mkheaders(headers), b, Response(0; body=responsebody), url, parent, context) r.response.request = r return r end """ "request-target" per https://tools.ietf.org/html/rfc7230#section-5.3 """ resource(uri::URI) = string( isempty(uri.path) ? "/" : uri.path, !isempty(uri.query) ? "?" : "", uri.query, !isempty(uri.fragment) ? "#" : "", uri.fragment) mkheaders(h::Headers) = h function mkheaders(h, headers=Vector{Header}(undef, length(h)))::Headers # validation for (i, head) in enumerate(h) head isa String && throw(ArgumentError("header must be passed as key => value pair: `$head`")) length(head) != 2 && throw(ArgumentError("invalid header key-value pair: $head")) headers[i] = SubString(string(head[1])) => SubString(string(head[2])) end return headers end method(r::Request) = getfield(r, :method) target(r::Request) = getfield(r, :target) url(r::Request) = getfield(r, :url) headers(r::Request) = getfield(r, :headers) body(r::Request) = getfield(r, :body) # HTTP Message state and type queries """ issafe(::Request) https://tools.ietf.org/html/rfc7231#section-4.2.1 """ issafe(r::Request) = issafe(r.method) issafe(method) = method in ["GET", "HEAD", "OPTIONS", "TRACE"] """ iserror(::Response) Does this `Response` have an error status? """ iserror(r::Response) = iserror(r.status) iserror(status::Integer) = status != 0 && status != 100 && status != 101 && (status < 200 || status >= 300) && !isredirect(status) """ isredirect(::Response) Does this `Response` have a redirect status? """ isredirect(r::Response) = isredirect(r.status) isredirect(r::Request) = allow_redirects(r) && !redirectlimitreached(r) isredirect(status::Integer) = status in (301, 302, 303, 307, 308) # whether the redirect limit has been reached for a given request # set in the RedirectRequest layer once the limit is reached redirectlimitreached(r::Request) = get(r.context, :redirectlimitreached, false) allow_redirects(r::Request) = get(r.context, :allow_redirects, false) """ isidempotent(::Request) https://tools.ietf.org/html/rfc7231#section-4.2.2 """ isidempotent(r::Request) = isidempotent(r.method) isidempotent(method) = issafe(method) || method in ["PUT", "DELETE"] retry_non_idempotent(r::Request) = get(r.context, :retry_non_idempotent, false) allow_retries(r::Request) = get(r.context, :allow_retries, false) nothing_written(r::Request) = get(r.context, :nothingwritten, false) # whether the retry limit has been reached for a given request # set in the RetryRequest layer once the limit is reached retrylimitreached(r::Request) = get(r.context, :retrylimitreached, false) """ retryable(::Request) Whether a `Request` is eligible to be retried. """ function retryable end supportsmark(x) = false supportsmark(x::T) where {T <: IO} = length(Base.methods(mark, Tuple{T}, parentmodule(T))) > 0 || hasfield(T, :mark) # request body is retryable if it was provided as "bytes", an AbstractDict or NamedTuple, # or a chunked array of "bytes"; OR if it supports mark() and is marked retryablebody(r::Request) = (isbytes(r.body) || r.body isa Union{AbstractDict, NamedTuple} || (r.body isa Vector && all(isbytes, r.body)) || (supportsmark(r.body) && ismarked(r.body))) # request is retryable if the body is retryable, the user is allowing retries at all, # we haven't reached the retry limit, and either nothing has been written yet or # the request is idempotent or the user has explicitly allowed non-idempotent retries retryable(r::Request) = retryablebody(r) && allow_retries(r) && !retrylimitreached(r) && (nothing_written(r) || isidempotent(r) || retry_non_idempotent(r)) retryable(r::Response) = retryable(r.status) retryable(status) = status in (403, 408, 409, 429, 500, 502, 503, 504, 599) """ ischunked(::Message) Does the `Message` have a "Transfer-Encoding: chunked" header? """ ischunked(m) = any(h->(field_name_isequal(h[1], "transfer-encoding") && endswith(lowercase(h[2]), "chunked")), m.headers) """ headerscomplete(::Message) Have the headers been read into this `Message`? """ headerscomplete(r::Response) = r.status != 0 && r.status != 100 headerscomplete(r::Request) = r.method != "" """ "The presence of a message body in a response depends on both the request method to which it is responding and the response status code. Responses to the HEAD request method never include a message body []. 2xx (Successful) responses to a CONNECT request method (Section 4.3.6 of [RFC7231]) switch to tunnel mode instead of having a message body. All 1xx (Informational), 204 (No Content), and 304 (Not Modified) responses do not include a message body. All other responses do include a message body, although the body might be of zero length." [RFC7230 3.3](https://tools.ietf.org/html/rfc7230#section-3.3) """ bodylength(r::Response)::Int = r.request.method == "HEAD" ? 0 : ischunked(r) ? unknown_length : r.status in [204, 304] ? 0 : (l = header(r, "Content-Length")) != "" ? parse(Int, l) : unknown_length """ "The presence of a message body in a request is signaled by a Content-Length or Transfer-Encoding header field. Request message framing is independent of method semantics, even if the method does not define any use for a message body." [RFC7230 3.3](https://tools.ietf.org/html/rfc7230#section-3.3) """ bodylength(r::Request)::Int = ischunked(r) ? unknown_length : parse(Int, header(r, "Content-Length", "0")) # HTTP header-fields Base.getindex(m::Message, k) = header(m, k) """ Are `field-name`s `a` and `b` equal? [HTTP `field-name`s](https://tools.ietf.org/html/rfc7230#section-3.2) are ASCII-only and case-insensitive. """ field_name_isequal(a, b) = ascii_lc_isequal(a, b) """ HTTP.header(::Message, key [, default=""]) -> String Get header value for `key` (case-insensitive). """ header(m::Message, k, d="") = header(m.headers, k, d) header(h::Headers, k::AbstractString, d="") = getbyfirst(h, k, k => d, field_name_isequal)[2] """ HTTP.headers(m::Message, key) -> Vector{String} Get all headers with key `k` or empty if none """ headers(h::Headers, k::AbstractString) = map(x -> x[2], filter(x -> field_name_isequal(x[1], k), h)) headers(m::Message, k::AbstractString) = headers(headers(m), k) """ HTTP.hasheader(::Message, key) -> Bool Does header value for `key` exist (case-insensitive)? """ hasheader(m, k::AbstractString) = header(m, k) != "" """ HTTP.hasheader(::Message, key, value) -> Bool Does header for `key` match `value` (both case-insensitive)? """ hasheader(m, k::AbstractString, v::AbstractString) = field_name_isequal(header(m, k), lowercase(v)) """ HTTP.headercontains(::Message, key, value) -> Bool Does the header for `key` (interpreted as comma-separated list) contain `value` (both case-insensitive)? """ headercontains(m, k::AbstractString, v::AbstractString) = any(field_name_isequal.(strip.(split(header(m, k), ",")), v)) """ HTTP.setheader(::Message, key => value) Set header `value` for `key` (case-insensitive). """ setheader(m::Message, v) = setheader(m.headers, v) setheader(h::Headers, v::Header) = setbyfirst(h, v, field_name_isequal) setheader(h::Headers, v::Pair) = setbyfirst(h, Header(SubString(v.first), SubString(v.second)), field_name_isequal) """ defaultheader!(::Message, key => value) Set header `value` in message for `key` if it is not already set. """ function defaultheader!(m, v::Pair) # return nothing as default to allow users passing "" as empty header # and not being overwritten by default headers if header(m, first(v), nothing) === nothing setheader(m, v) end return end """ HTTP.appendheader(::Message, key => value) Append a header value to `message.headers`. If `key` is the same as the previous header, the `value` is [appended to the value of the previous header with a comma delimiter](https://stackoverflow.com/a/24502264) `Set-Cookie` headers are not comma-combined because [cookies often contain internal commas](https://tools.ietf.org/html/rfc6265#section-3). """ function appendheader(m::Message, header::Header) c = m.headers k,v = header if k != "Set-Cookie" && length(c) > 0 && k == c[end][1] c[end] = c[end][1] => string(c[end][2], ", ", v) else push!(m.headers, header) end return end """ HTTP.removeheader(::Message, key) Remove header for `key` (case-insensitive). """ removeheader(m::Message, header::String) = rmkv(m.headers, header, field_name_isequal) # HTTP payload body function payload(m::Message)::Vector{UInt8} enc = lowercase(first(split(header(m, "Transfer-Encoding"), ", "))) return enc in ["", "identity", "chunked"] ? m.body : decode(m, enc) end payload(m::Message, ::Type{String}) = hasheader(m, "Content-Type", "ISO-8859-1") ? iso8859_1_to_utf8(payload(m)) : String(payload(m)) """ HTTP.decode(r::Union{Request, Response}) -> Vector{UInt8} For a gzip encoded request/response body, decompress it and return the decompressed body. """ function decode(m::Message, encoding::String="gzip")::Vector{UInt8} if encoding == "gzip" return transcode(GzipDecompressor, m.body) end return m.body end # Writing HTTP Messages to IO streams Base.write(io::IO, v::HTTPVersion) = write(io, "HTTP/", string(v.major), ".", string(v.minor)) """ writestartline(::IO, ::Message) e.g. `"GET /path HTTP/1.1\\r\\n"` or `"HTTP/1.1 200 OK\\r\\n"` """ function writestartline(io::IO, r::Request) return write(io, r.method, " ", r.target, " ", r.version, "\r\n") end function writestartline(io::IO, r::Response) return write(io, r.version, " ", string(r.status), " ", StatusCodes.statustext(r.status), "\r\n") end """ writeheaders(::IO, ::Message) Write `Message` start line and a line for each "name: value" pair and a trailing blank line. """ writeheaders(io::IO, m::Message) = writeheaders(io, m, IOBuffer()) writeheaders(io::Connection, m::Message) = writeheaders(io, m, io.writebuffer) function writeheaders(io::IO, m::Message, buf::IOBuffer) writestartline(buf, m) for (name, value) in m.headers # match curl convention of not writing empty headers !isempty(value) && write(buf, name, ": ", value, "\r\n") end write(buf, "\r\n") nwritten = write(io, take!(buf)) return nwritten end """ write(::IO, ::Message) Write start line, headers and body of HTTP Message. """ function Base.write(io::IO, m::Message) nwritten = writeheaders(io, m) nwritten += write(io, m.body) return nwritten end function Base.String(m::Message) io = IOBuffer() write(io, m) return String(take!(io)) end # Reading HTTP Messages from IO streams """ readheaders(::IO, ::Message) Read headers (and startline) from an `IO` stream into a `Message` struct. Throw `EOFError` if input is incomplete. """ function readheaders(io::IO, message::Message) bytes = String(IOExtras.readuntil(io, find_end_of_header)) bytes = parse_start_line!(bytes, message) parse_header_fields!(bytes, message) return end parse_start_line!(bytes, r::Response) = parse_status_line!(bytes, r) parse_start_line!(bytes, r::Request) = parse_request_line!(bytes, r) function parse_header_fields!(bytes::SubString{String}, m::Message) h, bytes = parse_header_field(bytes) while !(h === Parsers.emptyheader) appendheader(m, h) h, bytes = parse_header_field(bytes) end return end """ Read chunk-size from an `IO` stream. After the final zero size chunk, read trailers into a `Message` struct. """ function readchunksize(io::IO, message::Message)::Int n = parse_chunk_size(IOExtras.readuntil(io, find_end_of_chunk_size)) if n == 0 bytes = IOExtras.readuntil(io, find_end_of_trailer) if bytes[2] != UInt8('\n') parse_header_fields!(SubString(String(bytes)), message) end end return n end # Debug message printing """ set_show_max(x) Set the maximum number of body bytes to be displayed by `show(::IO, ::Message)` """ set_show_max(x) = BODY_SHOW_MAX[] = x const BODY_SHOW_MAX = Ref(1000) """ bodysummary(bytes) The first chunk of the Message Body (for display purposes). """ bodysummary(body) = isbytes(body) ? view(bytes(body), 1:min(nbytes(body), BODY_SHOW_MAX[])) : "[Message Body was streamed]" bodysummary(body::Union{AbstractDict, NamedTuple}) = URIs.escapeuri(body) function bodysummary(body::Form) if length(body.data) == 1 && isa(body.data[1], IOBuffer) return body.data[1].data[1:body.data[1].ptr-1] end return "[Message Body was streamed]" end function compactstartline(m::Message) b = IOBuffer() writestartline(b, m) return strip(String(take!(b))) end # temporary replacement for isvalid(String, s), until the # latter supports subarrays (JuliaLang/julia#36047): isvalidstr(s) = ccall(:u8_isvalid, Int32, (Ptr{UInt8}, Int), s, sizeof(s)) ≠ 0 function Base.show(io::IO, m::Message) if get(io, :compact, false) print(io, compactstartline(m)) if m isa Response print(io, " <= (", compactstartline(m.request::Request), ")") end return end println(io, typeof(m), ":") println(io, "\"\"\"") # Mask the following (potentially) sensitive headers with "******": # - Authorization # - Proxy-Authorization # - Cookie # - Set-Cookie header_str = sprint(writeheaders, m) header_str = replace(header_str, r"(*CRLF)^((?>(?>Proxy-)?Authorization|(?>Set-)?Cookie): ).+$"mi => s"\1******") write(io, header_str) summary = bodysummary(m.body) validsummary = isvalidstr(summary) validsummary && write(io, summary) if !validsummary || something(nbytes(m.body), 0) > length(summary) println(io, "\n⋮\n$(nbytes(m.body))-byte body") end print(io, "\"\"\"") return end function statustext(status) Base.depwarn("`Messages.statustext` is deprecated, use `StatusCodes.statustext` instead.", :statustext) return StatusCodes.statustext(status) end URIs.queryparams(r::Request) = URIs.queryparams(URI(r.target)) URIs.queryparams(r::Response) = isnothing(r.request) ? nothing : URIs.queryparams(r.request) end # module Messages -/opt/julia/packages/HTTP/sJD5V/src/cookies.jlG# based on go implementation in src/net/http/cookie.go # Copyright (c) 2009 The Go Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. module Cookies export Cookie, CookieJar, cookies, stringify, getcookies!, setcookies!, addcookie! import Base: == using Dates, URIs, LoggingExtras, Sockets using ..IOExtras, ..Parsers, ..Messages @enum SameSite SameSiteDefaultMode=1 SameSiteLaxMode SameSiteStrictMode SameSiteNoneMode """ Cookie() Cookie(; kwargs...) Cookie(name, value; kwargs...) A `Cookie` represents an HTTP cookie as sent in the `"Set-Cookie"` header of an HTTP response or the Cookie header of an HTTP request. Supported fields (which can be set using keyword arguments) include: * `name::String`: name of the cookie * `value::String`: value of the cookie * `path::String`: applicable path for the cookie * `domain::String`: applicable domain for the cookie * `expires::Dates.DateTime`: when the cookie should expire * `maxage::Int`: `maxage == 0` means no max age, `maxage < 0` means delete cookie now, `maxage > 0` means the # of seconds until expiration * `secure::Bool`: secure cookie attribute * `httponly::Bool`: httponly cookie attribute * `hostonly::Bool`: hostonly cookie attribute * `samesite::Bool`: SameSite cookie attribute See [IETF RFC 6265](http://tools.ietf.org/html/rfc6265) for details. The string representation of a cookie is generated by calling [`stringify(cookie; isrequest=true)`](@ref), where `isrequest=true` will only include the `name=value` pair for requests, and if `false`, will generate the `"Set-Cookie"` representation for a response header. A `Vector{Cookie}` can be retrieved from a `Request`/`Response` object by calling [`Cookies.cookies(r)`](@ref). A `Cookie` can be added to a `Request`/`Response` object by calling [`Cookies.addcookie!(r, cookie)`](@ref). """ mutable struct Cookie name::String value::String path::String # optional domain::String # optional expires::Dates.DateTime # optional rawexpires::String # for reading cookies only creation::Dates.DateTime lastaccess::Dates.DateTime # MaxAge=0 means no 'Max-Age' attribute specified. # MaxAge<0 means delete cookie now, equivalently 'Max-Age: 0' # MaxAge>0 means Max-Age attribute present and given in seconds maxage::Int secure::Bool httponly::Bool persistent::Bool hostonly::Bool samesite::SameSite raw::String unparsed::Vector{String} # Raw text of unparsed attribute-value pairs end function Cookie(cookie::Cookie; kwargs...) for (k, v) in kwargs setfield!(cookie, k, convert(fieldtype(Cookie, k), v)) end return cookie end Cookie(; kwargs...) = Cookie(Cookie("", "", ""); kwargs...) Cookie(name, value, raw=""; args...) = Cookie(Cookie( String(name), String(value), "", "", Dates.DateTime(1), "", Dates.DateTime(1), Dates.DateTime(1), 0, false, false, false, false, SameSiteDefaultMode, String(raw), String[]); args...) Base.isequal(a::Cookie, b::Cookie) = a.name == b.name && a.path == b.path && a.domain == b.domain Base.hash(x::Cookie, h::UInt) = hash(x.name, hash(x.path, hash(x.domain, h))) id(c::Cookie) = "$(c.domain);$(c.path);$(c.name)" ==(a::Cookie,b::Cookie) = (a.name == b.name) && (a.value == b.value) && (a.path == b.path) && (a.domain == b.domain) && (a.expires == b.expires) && (a.creation == b.creation) && (a.lastaccess == b.lastaccess) && (a.maxage == b.maxage) && (a.secure == b.secure) && (a.httponly == b.httponly) && (a.persistent == b.persistent) && (a.hostonly == b.hostonly) && (a.samesite == b.samesite) """ stringify(cookie::Cookie, isrequest=true) Generate the string representation of a cookie. By default, `isrequest=true`, and only the `name=value` pair will be included in the cookie string. For `isrequest=false`, the other cookie attributes will be included, `;`-separated, for use in a "Set-Cookie" header. """ function stringify(c::Cookie, isrequest::Bool=true) nm = strip(c.name) !iscookienamevalid(nm) && return "" io = IOBuffer() write(io, sanitizeCookieName(nm), '=', sanitizeCookieValue(c.value)) if !isrequest length(c.path) > 0 && write(io, "; Path=", sanitizeCookiePath(c.path)) length(c.domain) > 0 && validCookieDomain(c.domain) && write(io, "; Domain=", c.domain[1] == '.' ? SubString(c.domain, 2) : c.domain) validCookieExpires(c.expires) && write(io, "; Expires=", Dates.format(c.expires, Dates.RFC1123Format), " GMT") c.maxage > 0 && write(io, "; Max-Age=", string(c.maxage)) c.maxage < 0 && write(io, "; Max-Age=0") c.httponly && write(io, "; HttpOnly") # c.hostonly && write(io, "; HostOnly") c.secure && write(io, "; Secure") if c.samesite == SameSiteLaxMode write(io, "; SameSite=Lax") elseif c.samesite == SameSiteStrictMode write(io, "; SameSite=Strict") elseif c.samesite == SameSiteNoneMode write(io, "; SameSite=None") end end return String(take!(io)) end function stringify(cookiestring::AbstractString, cookies::Vector{Cookie}, isrequest::Bool=true) io = IOBuffer() if !isempty(cookiestring) write(io, cookiestring) if !isempty(cookies) if endswith(rstrip(cookiestring, [' ']), ";") cookiestring[end] == ';' && write(io, ' ') else write(io, "; ") end end end len = length(cookies) for (i, cookie) in enumerate(cookies) write(io, stringify(cookie, isrequest), ifelse(i == len, "", "; ")) end return String(take!(io)) end """ Cookies.addcookie!(r::Union{HTTP.Request, HTTP.Response}, c::Cookie) Convenience function for adding a single cookie to a request or response object. For requests, the cookie will be stringified and concatenated to any existing "Cookie" header. For responses, an additional "Set-Cookie" header will be appended. """ function addcookie! end function addcookie!(r::Request, c::Cookie) cstr = stringify(c) chead = header(r, "Cookie", nothing) if chead !== nothing setheader(r, "Cookie" => "$chead; $cstr") else appendheader(r, SubString("Cookie") => SubString(cstr)) end return r end function addcookie!(r::Response, c::Cookie) appendheader(r, SubString("Set-Cookie") => SubString(stringify(c, false))) return r end validcookiepathbyte(b) = (' ' <= b < '\x7f') && b != ';' validcookievaluebyte(b) = (' ' <= b < '\x7f') && b != '"' && b != ';' && b != '\\' function parsecookievalue(raw, allowdoublequote::Bool) if allowdoublequote && length(raw) > 1 && raw[1] == '"' && raw[end] == '"' raw = raw[2:end-1] end for i = eachindex(raw) !validcookievaluebyte(raw[i]) && return "", false end return raw, true end iscookienamevalid(raw) = raw == "" ? false : any(isurlchar, raw) gmtformat(::DateFormat{S,T}) where {S,T} = Dates.DateFormat(string(S, " G\\MT")) const AlternateRFC1123GMTFormat = gmtformat(dateformat"e, dd-uuu-yyyy HH:MM:SS") const RFC1123GMTFormat = gmtformat(Dates.RFC1123Format) # readSetCookies parses all "Set-Cookie" values from # the header h and returns the successfully parsed Cookies. function readsetcookies(h::Headers) result = Cookie[] for line in headers(h, "Set-Cookie") parts = split(strip(line), ';'; keepempty=false) if length(parts) == 1 && parts[1] == "" continue end part = strip(parts[1]) j = findfirst(isequal('='), part) if j !== nothing name, val = SubString(part, 1:j-1), SubString(part, j+1) else name, val = part, "" end !iscookienamevalid(name) && continue val, ok = parsecookievalue(val, true) !ok && continue c = Cookie(name, val, line) for i = 2:length(parts) part = strip(parts[i]) isempty(part) && continue j = findfirst(isequal('='), part) if j !== nothing attr, val = SubString(part, 1:j-1), SubString(part, j+1) else attr, val = part, "" end !isascii(attr) && continue lowerattr = lowercase(attr) val, ok = parsecookievalue(val, false) if !ok push!(c.unparsed, part) continue end if lowerattr == "samesite" if !isascii(val) c.samesite = SameSiteDefaultMode continue end val = lowercase(val) if val == "lax" c.samesite = SameSiteLaxMode elseif val == "strict" c.samesite = SameSiteStrictMode elseif val == "none" c.samesite = SameSiteNoneMode else c.samesite = SameSiteDefaultMode end elseif lowerattr == "secure" c.secure = true elseif lowerattr == "httponly" c.httponly = true # elseif lowerattr == "hostonly" # c.hostonly = true elseif lowerattr == "domain" c.domain = val elseif lowerattr == "max-age" try secs = parse(Int, val) val[1] == '0' && continue c.maxage = max(secs, -1) catch continue end elseif lowerattr == "expires" c.rawexpires = val try c.expires = Dates.DateTime(val, RFC1123GMTFormat) catch try c.expires = Dates.DateTime(val, AlternateRFC1123GMTFormat) catch continue end end elseif lowerattr == "path" c.path = val else push!(c.unparsed, parts[i]) end end push!(result, c) end return result end readsetcookies(h) = readsetcookies(mkheaders(h)) function isIP(host) try Base.parse(IPAddr, host) return true catch e isa(e, ArgumentError) && return false rethrow(e) end end """ HTTP.cookies(r::Union{Request, Response}) -> Vector{Cookie} Return a list of cookies, if any, parsed from the request "Cookie" or response "Set-Cookie" headers. """ function cookies end cookies(r::Response) = readsetcookies(r.headers) cookies(r::Request) = readcookies(r.headers, "") # readCookies parses all "Cookie" values from the header h and # returns the successfully parsed Cookies. # if filter isn't empty, only cookies of that name are returned function readcookies(h::Headers, filter::String="") result = Cookie[] for line in headers(h, "Cookie") for part in split(strip(line), ';'; keepempty=false) part = strip(part) length(part) <= 1 && continue j = findfirst(isequal('='), part) if j !== nothing name, val = part[1:j-1], part[j+1:end] else name, val = part, "" end !iscookienamevalid(name) && continue filter != "" && filter != name && continue val, ok = parsecookievalue(val, true) !ok && continue push!(result, Cookie(name, val, line)) end end return result end readcookies(h, f) = readcookies(mkheaders(h), f) # validCookieExpires returns whether v is a valid cookie expires-value. function validCookieExpires(dt) # IETF RFC 6265 Section 5.1.1.5, the year must not be less than 1601 return Dates.year(dt) >= 1601 end # validCookieDomain returns whether v is a valid cookie domain-value. function validCookieDomain(v::String) isCookieDomainName(v) && return true isIP(v) && !occursin(":", v) && return true return false end # isCookieDomainName returns whether s is a valid domain name or a valid # domain name with a leading dot '.'. It is almost a direct copy of # package net's isDomainName. function isCookieDomainName(s::String) length(s) == 0 && return false length(s) > 255 && return false s = s[1] == '.' ? s[2:end] : s last = '.' ok = false partlen = 0 for c in s if 'a' <= c <= 'z' || 'A' <= c <= 'Z' ok = true partlen += 1 elseif '0' <= c <= '9' partlen += 1 elseif c == '-' last == '.' && return false partlen += 1 elseif c == '.' (last == '.' || last == '-') && return false (partlen > 63 || partlen == 0) && return false partlen = 0 else return false end last = c end (last == '-' || partlen > 63) && return false return ok end sanitizeCookieName(n::String) = replace(replace(n, '\n'=>'-'), '\r'=>'-') sanitizeCookieName(n) = sanitizeCookieName(String(n)) # http:#tools.ietf.org/html/rfc6265#section-4.1.1 # cookie-value = *cookie-octet / ( DQUOTE *cookie-octet DQUOTE ) # cookie-octet = %x21 / %x23-2B / %x2D-3A / %x3C-5B / %x5D-7E # ; US-ASCII characters excluding CTLs, # ; whitespace DQUOTE, comma, semicolon, # ; and backslash # We loosen this as spaces and commas are common in cookie values # but we produce a quoted cookie-value in when value starts or ends # with a comma or space. # See https:#golang.org/issue/7243 for the discussion. function sanitizeCookieValue(v::String) v = String(filter(validcookievaluebyte, [Char(b) for b in bytes(v)])) length(v) == 0 && return v if contains(v, ' ') || contains(v, ',') return string('"', v, '"') end return v end sanitizeCookiePath(v) = filter(validcookiepathbyte, v) const normal_url_char = Bool[ #= 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel =# false, false, false, false, false, false, false, false, #= 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si =# false, true, false, false, true, false, false, false, #= 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb =# false, false, false, false, false, false, false, false, #= 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us =# false, false, false, false, false, false, false, false, #= 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' =# false, true, true, false, true, true, true, true, #= 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / =# true, true, true, true, true, true, true, true, #= 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 =# true, true, true, true, true, true, true, true, #= 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? =# true, true, true, true, true, true, true, false, #= 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G =# true, true, true, true, true, true, true, true, #= 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O =# true, true, true, true, true, true, true, true, #= 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W =# true, true, true, true, true, true, true, true, #= 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ =# true, true, true, true, true, true, true, true, #= 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g =# true, true, true, true, true, true, true, true, #= 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o =# true, true, true, true, true, true, true, true, #= 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w =# true, true, true, true, true, true, true, true, #= 120 x 121 y 122 z 123 { 124, 125 } 126 ~ 127 del =# true, true, true, true, true, true, true, false, ] @inline isurlchar(c) = c > '\u80' ? true : normal_url_char[Int(c) + 1] include("cookiejar.jl") end # module //opt/julia/packages/HTTP/sJD5V/src/cookiejar.jl-""" CookieJar() A thread-safe object for storing cookies returned in "Set-Cookie" response headers. Keyed by appropriate host from the original request made. Can be created manually and passed like `HTTP.get(url; cookiejar=mycookiejar)` to avoid using the default global `CookieJar`. The 2 main functions for interacting with a `CookieJar` are [`Cookies.getcookies!`](@ref), which returns a `Vector{Cookie}` for a given url (and will remove expired cookies from the jar), and [`Cookies.setcookies!`](@ref), which will store "Set-Cookie" response headers in the cookie jar. """ struct CookieJar lock::ReentrantLock # map of host to cookies mapped by id(::Cookie) entries::Dict{String, Dict{String, Cookie}} end CookieJar() = CookieJar(ReentrantLock(), Dict{String, Dict{String, Cookie}}()) Base.empty!(c::CookieJar) = lock(() -> empty!(c.entries), c.lock) # shouldsend determines whether e's cookie qualifies to be included in a # request to host/path. It is the caller's responsibility to check if the # cookie is expired. function shouldsend(cookie::Cookie, https::Bool, host, path) return domainmatch(cookie, host) && pathmatch(cookie, path) && (https || !cookie.secure) end # domainMatch implements "domain-match" of RFC 6265 section 5.1.3. function domainmatch(cookie::Cookie, host) cookie.domain == host && return true return !cookie.hostonly && hasdotsuffix(host, cookie.domain) end # hasdotsuffix reports whether s ends in "."+suffix. function hasdotsuffix(s, suffix) return length(s) > length(suffix) && s[length(s)-length(suffix)] == '.' && s[(length(s)-length(suffix)+1):end] == suffix end # pathMatch implements "path-match" according to RFC 6265 section 5.1.4. function pathmatch(cookie::Cookie, requestpath) requestpath == cookie.path && return true if startswith(requestpath, cookie.path) if length(cookie.path) > 0 && cookie.path[end] == '/' return true # The "/any/" matches "/any/path" case. elseif length(requestpath) >= length(cookie.path) + 1 && requestpath[length(cookie.path)+1] == '/' return true # The "/any" matches "/any/path" case. end end return false end """ Cookies.getcookies!(jar::CookieJar, url::URI) Retrieve valid `Cookie`s from the `CookieJar` according to the provided `url`. Cookies will be returned as a `Vector{Cookie}`. Only cookies for `http` or `https` scheme in the url will be returned. Cookies will be checked according to the canonical host of the url and any cookie max age or expiration will be accounted for. Expired cookies will not be returned and will be removed from the cookie jar. """ function getcookies!(jar::CookieJar, url::URI, now::DateTime=Dates.now(Dates.UTC))::Vector{Cookie} cookies = Cookie[] if url.scheme != "http" && url.scheme != "https" return cookies end host = canonicalhost(url.host) host == "" && return cookies Base.@lock jar.lock begin !haskey(jar.entries, host) && return cookies entries = jar.entries[host] https = url.scheme == "https" path = url.path if path == "" path = "/" end modified = false expired = Cookie[] for (id, e) in entries if e.persistent && e.expires != DateTime(1) && e.expires < now @debugv 1 "Deleting expired cookie: $(e.name)" push!(expired, e) continue end if !shouldsend(e, https, host, path) continue end e.lastaccess = now @debugv 1 "Including cookie in request: $(e.name) to $(url.host)" push!(cookies, e) end for c in expired delete!(entries, id(c)) end end sort!(cookies; lt=(x, y) -> begin if length(x.path) != length(y.path) return length(x.path) > length(y.path) end if x.creation != y.creation return x.creation < y.creation end return x.name < y.name end) return cookies end """ Cookies.setcookies!(jar::CookieJar, url::URI, headers::Headers) Identify, "Set-Cookie" response headers from `headers`, parse the `Cookie`s, and store valid entries in the cookie `jar` according to the canonical host in `url`. Cookies can be retrieved from the `jar` via [`Cookies.getcookies!`](@ref). """ function setcookies!(jar::CookieJar, url::URI, headers::Headers) cookies = readsetcookies(headers) isempty(cookies) && return if url.scheme != "http" && url.scheme != "https" return end host = canonicalhost(url.host) host == "" && return defPath = defaultPath(url.path) now = Dates.now(Dates.UTC) Base.@lock jar.lock begin entries = get!(() -> Dict{String, Cookie}(), jar.entries, host) for c in cookies if c.path == "" || c.path[1] != '/' c.path = defPath end domainAndType!(jar, c, host) || continue cid = id(c) if c.maxage < 0 @goto remove elseif c.maxage > 0 c.expires = now + Dates.Second(c.maxage) c.persistent = true else if c.expires == DateTime(1) c.expires = endOfTime c.persistent = false else if c.expires < now @debugv 1 "Cookie expired: $(c.name)" @goto remove end c.persistent = true end end if haskey(entries, cid) old = entries[cid] c.creation = old.creation else c.creation = now end c.lastaccess = now entries[cid] = c continue @label remove delete!(entries, cid) end end return end function canonicalhost(host) if hasport(host) host, _, err = splithostport(host) err && return "" end if host[end] == '.' host = chop(host) end return isascii(host) ? lowercase(host) : "" end function hasport(host) colons = count(":", host) colons == 0 && return false colons == 1 && return true return host[1] == '[' && contains(host, "]:") end function defaultPath(path) if isempty(path) || path[1] != '/' return "/" end i = findlast('/', path) if i === nothing || i == 1 return "/" end return path[1:i] end const endOfTime = DateTime(9999, 12, 31, 23, 59, 59, 0) function domainAndType!(jar::CookieJar, c::Cookie, host::String) domain = c.domain if domain == "" # No domain attribute in the SetCookie header indicates a # host cookie. c.domain = host c.hostonly = true return true end if isIP(host) # RFC 6265 is not super clear here, a sensible interpretation # is that cookies with an IP address in the domain-attribute # are allowed. # RFC 6265 section 5.2.3 mandates to strip an optional leading # dot in the domain-attribute before processing the cookie. # # Most browsers don't do that for IP addresses, only curl # version 7.54) and and IE (version 11) do not reject a # Set-Cookie: a=1; domain=.127.0.0.1 # This leading dot is optional and serves only as hint for # humans to indicate that a cookie with "domain=.bbc.co.uk" # would be sent to every subdomain of bbc.co.uk. # It just doesn't make sense on IP addresses. # The other processing and validation steps in RFC 6265 just # collaps to: if host != domain c.domain = "" c.hostonly = false return false end # According to RFC 6265 such cookies should be treated as # domain cookies. # As there are no subdomains of an IP address the treatment # according to RFC 6265 would be exactly the same as that of # a host-only cookie. Contemporary browsers (and curl) do # allows such cookies but treat them as host-only cookies. # So do we as it just doesn't make sense to label them as # domain cookies when there is no domain; the whole notion of # domain cookies requires a domain name to be well defined. c.domain = host c.hostonly = true return true end # From here on: If the cookie is valid, it is a domain cookie (with # the one exception of a public suffix below). # See RFC 6265 section 5.2.3. if domain[1] == '.' domain = chop(domain; head=1, tail=0) end if length(domain) == 0 || domain[1] == '.' # Received either "Domain=." or "Domain=..some.thing", # both are illegal. c.domain = "" c.hostonly = false return false end if !isascii(domain) # Received non-ASCII domain, e.g. "perché.com" instead of "xn--perch-fsa.com" c.domain = "" c.hostonly = false return false end domain = lowercase(domain) if domain[end] == '.' # We received stuff like "Domain=www.example.com.". # Browsers do handle such stuff (actually differently) but # RFC 6265 seems to be clear here (e.g. section 4.1.2.3) in # requiring a reject. 4.1.2.3 is not normative, but # "Domain Matching" (5.1.3) and "Canonicalized Host Names" # (5.1.2) are. c.domain = "" c.hostonly = false return false end # The domain must domain-match host: www.mycompany.com cannot # set cookies for .ourcompetitors.com. if host != domain && !hasdotsuffix(host, domain) c.domain = "" c.hostonly = false return false end c.domain = domain c.hostonly = false return true end # SplitHostPort splits a network address of the form "host:port", # "host%zone:port", "[host]:port" or "[host%zone]:port" into host or # host%zone and port. # # A literal IPv6 address in hostport must be enclosed in square # brackets, as in "[::1]:80", "[::1%lo0]:80". # # See func Dial for a description of the hostport parameter, and host # and port results. function splithostport(hostport) j = k = 1 # The port starts after the last colon. i = findlast(':', hostport) if i === nothing return "", "", true end if hostport[1] == '[' # Expect the first ']' just before the last ':'. z = findfirst(']', hostport) if z === nothing return "", "", true end if z == length(hostport) return "", "", true elseif (z + 1) == i # expected else # Either ']' isn't followed by a colon, or it is # followed by a colon that is not the last one. return "", "", true end host = SubString(hostport, 2:(z-1)) j = 2 k = z + 1 # there can't be a '[' resp. ']' before these positions else host = SubString(hostport, 1:i-1) if contains(host, ":") return "", "", true end end len = length(hostport) if findfirst('[', SubString(hostport, j:len)) !== nothing return "", "", true end if findfirst(']', SubString(hostport, k:len)) !== nothing return "", "", true end port = SubString(hostport, (i+1):len) return host, port, false end-/opt/julia/packages/HTTP/sJD5V/src/Streams.jlw-module Streams export Stream, closebody, isaborted, setstatus, readall! using Sockets, LoggingExtras using ..IOExtras, ..Messages, ..Connections, ..Conditions, ..Exceptions import ..HTTP # for doc references mutable struct Stream{M <: Message, S <: IO} <: IO message::M stream::S writechunked::Bool readchunked::Bool warn_not_to_read_one_byte_at_a_time::Bool ntoread::Int nwritten::Int end """ Stream(::Request, ::IO) Creates a `HTTP.Stream` that wraps an existing `IO` stream. - `startwrite(::Stream)` sends the `Request` headers to the `IO` stream. - `write(::Stream, body)` sends the `body` (or a chunk of the body). - `closewrite(::Stream)` sends the final `0` chunk (if needed) and calls `closewrite` on the `IO` stream. - `startread(::Stream)` calls `startread` on the `IO` stream then reads and parses the `Response` headers. - `eof(::Stream)` and `readavailable(::Stream)` parse the body from the `IO` stream. - `closeread(::Stream)` reads the trailers and calls `closeread` on the `IO` stream. When the `IO` stream is a [`HTTP.Connections.Connection`](@ref), calling `closeread` releases the connection back to the connection pool for reuse. If a complete response has not been received, `closeread` throws `EOFError`. """ Stream(r::M, io::S) where {M, S} = Stream{M, S}(r, io, false, false, true, 0, -1) Messages.header(http::Stream, a...) = header(http.message, a...) setstatus(http::Stream, status) = (http.message.response.status = status) Messages.setheader(http::Stream, a...) = setheader(http.message.response, a...) Connections.getrawstream(http::Stream) = getrawstream(http.stream) Sockets.getsockname(http::Stream) = Sockets.getsockname(IOExtras.tcpsocket(getrawstream(http))) function Sockets.getpeername(http::Stream) # TODO: MbedTLS only forwards getsockname(::SSLContext) # so we use IOExtras.tcpsocket to reach into the MbedTLS internals # for now to keep compatibility with older MbedTLS versions. # return Sockets.getpeername(getrawstream(http)) return Sockets.getpeername(IOExtras.tcpsocket(getrawstream(http))) end IOExtras.isopen(http::Stream) = isopen(http.stream) # Writing HTTP Messages messagetowrite(http::Stream{<:Response}) = http.message.request::Request messagetowrite(http::Stream{<:Request}) = http.message.response IOExtras.iswritable(http::Stream) = iswritable(http.stream) function IOExtras.startwrite(http::Stream) if !iswritable(http.stream) startwrite(http.stream) end m = messagetowrite(http) if !hasheader(m, "Content-Length") && !hasheader(m, "Transfer-Encoding") && !hasheader(m, "Upgrade") && (m isa Request || (m.request.version >= v"1.1" && bodylength(m) > 0)) http.writechunked = true setheader(m, "Transfer-Encoding" => "chunked") else http.writechunked = ischunked(m) end n = writeheaders(http.stream, m) # nwritten starts at -1 so that we can tell if we've written anything yet http.nwritten = 0 # should not include headers return n end function Base.unsafe_write(http::Stream, p::Ptr{UInt8}, n::UInt) if n == 0 return 0 end if !iswritable(http) && isopen(http.stream) startwrite(http) end nw = if !http.writechunked unsafe_write(http.stream, p, n) else write(http.stream, string(n, base=16), "\r\n") + unsafe_write(http.stream, p, n) + write(http.stream, "\r\n") end http.nwritten += nw return nw end """ closebody(::Stream) Write the final `0` chunk if needed. """ function closebody(http::Stream) if http.writechunked http.writechunked = false @try Base.IOError write(http.stream, "0\r\n\r\n") end end function IOExtras.closewrite(http::Stream{<:Response}) if !iswritable(http) return end closebody(http) closewrite(http.stream) end function IOExtras.closewrite(http::Stream{<:Request}) if iswritable(http) closebody(http) closewrite(http.stream) end if hasheader(http.message, "Connection", "close") || hasheader(http.message, "Connection", "upgrade") || http.message.version < v"1.1" && !hasheader(http.message, "Connection", "keep-alive") @debugv 1 "✋ \"Connection: close\": $(http.stream)" close(http.stream) end end # Reading HTTP Messages IOExtras.isreadable(http::Stream) = isreadable(http.stream) Base.bytesavailable(http::Stream) = min(ntoread(http), bytesavailable(http.stream)) function IOExtras.startread(http::Stream) if !isreadable(http.stream) startread(http.stream) end readheaders(http.stream, http.message) handle_continue(http) http.readchunked = ischunked(http.message) http.ntoread = bodylength(http.message) return http.message end """ 100 Continue https://tools.ietf.org/html/rfc7230#section-5.6 https://tools.ietf.org/html/rfc7231#section-6.2.1 """ function handle_continue(http::Stream{<:Response}) if http.message.status == 100 @debugv 1 "✅ Continue: $(http.stream)" readheaders(http.stream, http.message) end end function handle_continue(http::Stream{<:Request}) if hasheader(http.message, "Expect", "100-continue") if !iswritable(http.stream) startwrite(http.stream) end @debugv 1 "✅ Continue: $(http.stream)" writeheaders(http.stream, Response(100)) end end function Base.eof(http::Stream) if !headerscomplete(http.message) startread(http) end if http.ntoread == 0 return true end return eof(http.stream) end @inline function ntoread(http::Stream) if !headerscomplete(http.message) startread(http) end # Find length of next chunk if http.ntoread == unknown_length && http.readchunked http.ntoread = readchunksize(http.stream, http.message) end return http.ntoread end @inline function update_ntoread(http::Stream, n) if http.ntoread != unknown_length http.ntoread -= n end if http.readchunked if http.ntoread == 0 http.ntoread = unknown_length end end @ensure http.ntoread >= 0 end function Base.readavailable(http::Stream, n::Int=typemax(Int)) ntr = ntoread(http) if ntr == 0 return UInt8[] end bytes = read(http.stream, min(n, ntr)) update_ntoread(http, length(bytes)) return bytes end Base.read(http::Stream, n::Integer) = readavailable(http, Int(n)) function Base.read(http::Stream, ::Type{UInt8}) if http.warn_not_to_read_one_byte_at_a_time @warn "Reading one byte at a time from HTTP.Stream is inefficient.\n" * "Use: io = BufferedInputStream(http::HTTP.Stream) instead.\n" * "See: https://github.com/BioJulia/BufferedStreams.jl" http.warn_not_to_read_one_byte_at_a_time = false end if ntoread(http) == 0 throw(EOFError()) end update_ntoread(http, 1) return read(http.stream, UInt8) end function http_unsafe_read(http::Stream, p::Ptr{UInt8}, n::UInt)::Int ntr = UInt(ntoread(http)) ntr == 0 && return 0 # If there is spare space in `p` # read two extra bytes # (`\r\n` at end ofchunk). unsafe_read(http.stream, p, min(n, ntr + (http.readchunked ? 2 : 0))) n = min(n, ntr) update_ntoread(http, n) return n end function Base.readbytes!(http::Stream, buf::AbstractVector{UInt8}, n=length(buf)) n > length(buf) && resize!(buf, n) return GC.@preserve buf http_unsafe_read(http, pointer(buf), UInt(n)) end function Base.unsafe_read(http::Stream, p::Ptr{UInt8}, n::UInt) nread = 0 while nread < n if eof(http) throw(EOFError()) end nread += http_unsafe_read(http, p + nread, n - nread) end nothing end function Base.readbytes!(http::Stream, buf::Base.GenericIOBuffer, n=bytesavailable(http)) p, nbmax = Base.alloc_request(buf, UInt(n)) nbmax < n && throw(ArgumentError("Unable to grow response stream IOBuffer $nbmax large enough for response body size: $n")) GC.@preserve buf unsafe_read(http, p, UInt(n)) # TODO: use `Base.notify_filled(buf, Int(n))` here, but only once it is identical to this: if buf.append buf.size += Int(n) else buf.ptr += Int(n) buf.size = max(buf.size, buf.ptr - 1) end return n end function Base.read(http::Stream, buf::Base.GenericIOBuffer=PipeBuffer()) readall!(http, buf) return take!(buf) end function readall!(http::Stream, buf::Base.GenericIOBuffer=PipeBuffer()) n = 0 if ntoread(http) == unknown_length while !eof(http) n += readbytes!(http, buf) end else # even if we know the length, we still need to read until eof # because Transfer-Encoding: chunked comes in piece-by-piece while !eof(http) n += readbytes!(http, buf, ntoread(http)) end end return n end function IOExtras.readuntil(http::Stream, f::Function) UInt(ntoread(http)) == 0 && return Connections.nobytes try bytes = IOExtras.readuntil(http.stream, f) update_ntoread(http, length(bytes)) return bytes catch ex ex isa EOFError || rethrow() # if we error, it means we didn't find what we were looking for # TODO: this seems very sketchy return UInt8[] end end """ isaborted(::Stream{<:Response}) Has the server signaled that it does not wish to receive the message body? "If [the response] indicates the server does not wish to receive the message body and is closing the connection, the client SHOULD immediately cease transmitting the body and close the connection." [RFC7230, 6.5](https://tools.ietf.org/html/rfc7230#section-6.5) """ function isaborted(http::Stream{<:Response}) if iswritable(http.stream) && iserror(http.message) && hasheader(http.message, "Connection", "close") @debugv 1 "✋ Abort on $(sprint(writestartline, http.message)): " * "$(http.stream)" @debugv 2 "✋ $(http.message)" return true end return false end Messages.isredirect(http::Stream{<:Response}) = isredirect(http.message) && isredirect(http.message.request) Messages.retryable(http::Stream{<:Response}) = retryable(http.message) && retryable(http.message.request) incomplete(http::Stream) = http.ntoread > 0 && (http.readchunked || http.ntoread != unknown_length) function IOExtras.closeread(http::Stream{<:Response}) if hasheader(http.message, "Connection", "close") # Close conncetion if server sent "Connection: close"... @debugv 1 "✋ \"Connection: close\": $(http.stream)" close(http.stream) # Error if Message is not complete... incomplete(http) && throw(EOFError()) else # Discard body bytes that were not read... @try Base.IOError EOFError while !eof(http) readavailable(http) end if incomplete(http) # Error if Message is not complete... close(http.stream) throw(EOFError()) elseif isreadable(http.stream) closeread(http.stream) end end return http.message end function IOExtras.closeread(http::Stream{<:Request}) if incomplete(http) # Error if Message is not complete... close(http.stream) throw(EOFError()) end if isreadable(http) closeread(http.stream) end end end #module Streams A/opt/julia/packages/HTTP/sJD5V/src/clientlayers/MessageRequest.jlumodule MessageRequest using URIs, LoggingExtras using ..IOExtras, ..Messages, ..Parsers, ..Exceptions using ..Messages, ..Parsers using ..Strings: HTTPVersion import ..DEBUG_LEVEL export messagelayer # like Messages.mkheaders, but we want to make a copy of user-provided headers # and avoid double copy when no headers provided (very common) mkreqheaders(::Nothing, ch) = Header[] mkreqheaders(headers::Headers, ch) = ch ? copy(headers) : headers mkreqheaders(h, ch) = mkheaders(h) """ messagelayer(handler) -> handler Construct a [`Request`](@ref) object from method, url, headers, and body. Hard-coded as the first layer in the request pipeline. """ function messagelayer(handler) return function makerequest(method::String, url::URI, headers, body; copyheaders::Bool=true, response_stream=nothing, http_version=HTTPVersion(1, 1), verbose=DEBUG_LEVEL[], kw...) req = Request(method, resource(url), mkreqheaders(headers, copyheaders), body; url=url, version=http_version, responsebody=response_stream) local resp start_time = time() try # if debugging, enable by wrapping request in custom logger logic resp = if verbose > 0 LoggingExtras.withlevel(Logging.Debug; verbosity=verbose) do handler(req; verbose, response_stream, kw...) end else handler(req; verbose, response_stream, kw...) end catch e if e isa CapturedException e = e.ex end if e isa StatusError resp = e.response end rethrow(e) finally dur = (time() - start_time) * 1000 req.context[:total_request_duration_ms] = dur if @isdefined(resp) && iserror(resp) && haskey(resp.request.context, :response_body) if isbytes(resp.body) resp.body = resp.request.context[:response_body] else write(resp.body, resp.request.context[:response_body]) end end end end end end # module MessageRequest B/opt/julia/packages/HTTP/sJD5V/src/clientlayers/RedirectRequest.jlmodule RedirectRequest using URIs, LoggingExtras using ..Messages, ..Pairs export redirectlayer, nredirects """ redirectlayer(handler) -> handler Redirects the request in the case of 3xx response status. """ function redirectlayer(handler) return function redirects(req; redirect::Bool=true, redirect_limit::Int=3, redirect_method=nothing, forwardheaders::Bool=true, response_stream=nothing, kw...) if !redirect || redirect_limit == 0 # no redirecting return handler(req; kw...) end req.context[:allow_redirects] = true count = 0 while true # Verify the url before making the request. Verification is done in # the redirect loop to also catch bad redirect URLs. verify_url(req.url) res = handler(req; kw...) if (count == redirect_limit || !isredirect(res) || (location = header(res, "Location")) == "") return res end # follow redirect oldurl = req.url url = resolvereference(req.url, location) method = newmethod(req.method, res.status, redirect_method) body = method == "GET" ? UInt8[] : req.body req = Request(method, resource(url), copy(req.headers), body; url=url, version=req.version, responsebody=response_stream, parent=res, context=req.context) if forwardheaders req.headers = filter(req.headers) do (header, _) # false return values are filtered out if header == "Host" return false elseif (header in SENSITIVE_HEADERS && !isdomainorsubdomain(url.host, oldurl.host)) return false elseif method == "GET" && header in ("Content-Type", "Content-Length") return false else return true end end else req.headers = Header[] end @debugv 1 "➡️ Redirect: $url" count += 1 if count == redirect_limit req.context[:redirectlimitreached] = true end end @assert false "Unreachable!" end end function nredirects(req) return req.parent === nothing ? 0 : (1 + nredirects(req.parent.request)) end const SENSITIVE_HEADERS = Set([ "Authorization", "Www-Authenticate", "Cookie", "Cookie2" ]) function isdomainorsubdomain(sub, parent) sub == parent && return true endswith(sub, parent) || return false return sub[length(sub)-length(parent)] == '.' end function verify_url(url::URI) if !(url.scheme in ("http", "https", "ws", "wss")) throw(ArgumentError("missing or unsupported scheme in URL (expected http(s) or ws(s)): $(url)")) end if isempty(url.host) throw(ArgumentError("missing host in URL: $(url)")) end end function newmethod(request_method, response_status, redirect_method) # using https://everything.curl.dev/http/redirects#get-or-post as a reference # also reference: https://github.com/curl/curl/issues/5237#issuecomment-618293609 if response_status == 307 || response_status == 308 # specific status codes that indicate an identical request should be made to new location return request_method elseif response_status == 303 # 303 means it's a new/different URI, so only GET allowed return "GET" elseif redirect_method == :same return request_method elseif redirect_method !== nothing && String(redirect_method) in ("GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS", "PATCH") return redirect_method elseif request_method == "HEAD" # Unless otherwise specified (e.g. with `redirect_method`), be conservative and keep the # same method, see: # # * # * # # Turning a HEAD request through a redirect may be undesired: # . return request_method end return "GET" end end # module RedirectRequest A/opt/julia/packages/HTTP/sJD5V/src/clientlayers/HeadersRequest.jl module HeadersRequest export headerslayer, setuseragent! using Base64, URIs, LoggingExtras using ..Messages, ..Forms, ..IOExtras, ..Sniff, ..Forms, ..Strings """ headerslayer(handler) -> handler Sets default expected headers. """ function headerslayer(handler) return function defaultheaders(req; iofunction=nothing, decompress=nothing, basicauth::Bool=true, detect_content_type::Bool=false, canonicalize_headers::Bool=false, kw...) headers = req.headers ## basicauth if basicauth userinfo = unescapeuri(req.url.userinfo) if !isempty(userinfo) && !hasheader(headers, "Authorization") @debugv 1 "Adding Authorization: Basic header." setheader(headers, "Authorization" => "Basic $(base64encode(userinfo))") end end ## content type detection if detect_content_type && (!hasheader(headers, "Content-Type") && !isa(req.body, Form) && isbytes(req.body)) sn = sniff(bytes(req.body)) setheader(headers, "Content-Type" => sn) @debugv 1 "setting Content-Type header to: $sn" end ## default headers if isempty(req.url.port) || (req.url.scheme == "http" && req.url.port == "80") || (req.url.scheme == "https" && req.url.port == "443") hostheader = req.url.host else hostheader = req.url.host * ":" * req.url.port end defaultheader!(headers, "Host" => hostheader) defaultheader!(headers, "Accept" => "*/*") if USER_AGENT[] !== nothing defaultheader!(headers, "User-Agent" => USER_AGENT[]) end if !hasheader(headers, "Content-Length") && !hasheader(headers, "Transfer-Encoding") && !hasheader(headers, "Upgrade") l = nbytes(req.body) if l !== nothing setheader(headers, "Content-Length" => string(l)) elseif req.method == "GET" && iofunction isa Function setheader(headers, "Content-Length" => "0") end end if !hasheader(headers, "Content-Type") && req.body isa Form && req.method in ("POST", "PUT", "PATCH") # "Content-Type" => "multipart/form-data; boundary=..." setheader(headers, content_type(req.body)) elseif !hasheader(headers, "Content-Type") && (req.body isa Union{AbstractDict, NamedTuple}) && req.method in ("POST", "PUT", "PATCH") setheader(headers, "Content-Type" => "application/x-www-form-urlencoded") end if decompress === nothing || decompress defaultheader!(headers, "Accept-Encoding" => "gzip") end ## canonicalize headers if canonicalize_headers req.headers = canonicalizeheaders(headers) end res = handler(req; iofunction, decompress, kw...) if canonicalize_headers res.headers = canonicalizeheaders(res.headers) end return res end end canonicalizeheaders(h::T) where {T} = T([tocameldash(k) => v for (k,v) in h]) const USER_AGENT = Ref{Union{String, Nothing}}("HTTP.jl/$VERSION") """ setuseragent!(x::Union{String, Nothing}) Set the default User-Agent string to be used in each HTTP request. Can be manually overridden by passing an explicit `User-Agent` header. Setting `nothing` will prevent the default `User-Agent` header from being passed. """ function setuseragent!(x::Union{String, Nothing}) USER_AGENT[] = x return end end # module @/opt/julia/packages/HTTP/sJD5V/src/clientlayers/CookieRequest.jl$module CookieRequest using Dates, LoggingExtras, URIs using ..Cookies, ..Messages, ..Strings # default global cookie jar const COOKIEJAR = CookieJar() export cookielayer, COOKIEJAR """ cookielayer(handler) -> handler Check for host-appropriate cookies to include in the outgoing request from the `cookiejar` keyword argument (by default, a global cookiejar is used). Store "Set-Cookie" cookies from the response headers. """ function cookielayer(handler) return function managecookies(req::Request; cookies=true, cookiejar::CookieJar=COOKIEJAR, kw...) if cookies === true || (cookies isa AbstractDict && !isempty(cookies)) url = req.url cookiestosend = Cookies.getcookies!(cookiejar, url) if !(cookies isa Bool) for (name, value) in cookies push!(cookiestosend, Cookie(name, value)) end end if !isempty(cookiestosend) existingcookie = header(req.headers, "Cookie") if existingcookie != "" && haskey(req.context, :includedCookies) # this is a redirect where we previously included cookies # we want to filter those out to avoid duplicate cookie sending # and the case where a cookie was set to expire from the 1st request previouscookies = Cookies.cookies(req) previouslyincluded = req.context[:includedCookies] filtered = filter(x -> !(x.name in previouslyincluded), previouscookies) existingcookie = stringify("", filtered) end setheader(req.headers, "Cookie" => stringify(existingcookie, cookiestosend)) req.context[:includedCookies] = map(x -> x.name, cookiestosend) end res = handler(req; kw...) Cookies.setcookies!(cookiejar, url, res.headers) return res else # skip return handler(req; kw...) end end end end # module CookieRequest A/opt/julia/packages/HTTP/sJD5V/src/clientlayers/TimeoutRequest.jlKmodule TimeoutRequest using ..Connections, ..Streams, ..Exceptions, ..Messages using LoggingExtras, ConcurrentUtilities using ..Exceptions: current_exceptions_to_string export timeoutlayer """ timeoutlayer(handler) -> handler Close the `HTTP.Stream` if no data has been received for `readtimeout` seconds. """ function timeoutlayer(handler) return function timeouts(stream::Stream; readtimeout::Int=0, logerrors::Bool=false, logtag=nothing, kw...) if readtimeout <= 0 # skip return handler(stream; logerrors=logerrors, kw...) end return try try_with_timeout(readtimeout, Response) do timedout handler(stream; logerrors=logerrors, logtag=logtag, timedout=timedout, kw...) end catch e if e isa ConcurrentUtilities.TimeoutException req = stream.message.request req.context[:timeout_errors] = get(req.context, :timeout_errors, 0) + 1 if logerrors @error current_exceptions_to_string() type=Symbol("HTTP.TimeoutError") method=req.method url=req.url context=req.context timeout=readtimeout logtag=logtag end e = Exceptions.TimeoutError(readtimeout) end rethrow(e) end end end end # module TimeoutRequest C/opt/julia/packages/HTTP/sJD5V/src/clientlayers/ExceptionRequest.jl%module ExceptionRequest export exceptionlayer using ..IOExtras, ..Messages, ..Exceptions """ exceptionlayer(handler) -> handler Throw a `StatusError` if the request returns an error response status. """ function exceptionlayer(handler) return function exceptions(stream; status_exception::Bool=true, timedout=nothing, logerrors::Bool=false, logtag=nothing, kw...) res = handler(stream; timedout=timedout, logerrors=logerrors, logtag=logtag, kw...) if status_exception && iserror(res) req = res.request req.context[:status_errors] = get(req.context, :status_errors, 0) + 1 e = StatusError(res.status, req.method, req.target, res) throw(e) else return res end end end end # module ExceptionRequest ?/opt/julia/packages/HTTP/sJD5V/src/clientlayers/RetryRequest.jlmodule RetryRequest using Sockets, LoggingExtras, MbedTLS, OpenSSL, ExceptionUnwrapping using ..IOExtras, ..Messages, ..Strings, ..ExceptionRequest, ..Exceptions export retrylayer FALSE(x...) = false """ retrylayer(handler) -> handler Retry the request if it throws a recoverable exception. `Base.retry` and `Base.ExponentialBackOff` implement a randomised exponentially increasing delay is introduced between attempts to avoid exacerbating network congestion. By default, requests that have a retryable body, where the request wasn't written or is idempotent will be retried. If the request is made and a response is received with a status code of 403, 408, 409, 429, or 5xx, the request will be retried. `retries` controls the # of total retries that will be attempted. `retry_check` allows passing a custom retry check in the case where the default retry check _wouldn't_ retry, if `retry_check` returns true, then the request will be retried anyway. """ function retrylayer(handler) return function manageretries(req::Request; retry::Bool=true, retries::Int=4, retry_delays=ExponentialBackOff(n = retries, factor=3.0), retry_check=FALSE, retry_non_idempotent::Bool=false, kw...) if !retry || retries == 0 # no retry return handler(req; kw...) end req.context[:allow_retries] = true req.context[:retryattempt] = 0 if retry_non_idempotent req.context[:retry_non_idempotent] = true end req_body_is_marked = false if req.body isa IO && Messages.supportsmark(req.body) @debugv 2 "Marking request body stream" req_body_is_marked = true mark(req.body) end retryattempt = Ref(0) retry_request = Base.retry(handler, delays=retry_delays, check=(s, ex) -> begin retryattempt[] += 1 req.context[:retryattempt] = retryattempt[] retry = ( (isrecoverable(ex) && retryable(req)) || (retryablebody(req) && !retrylimitreached(req) && _retry_check(s, ex, req, retry_check)) ) if retryattempt[] == retries req.context[:retrylimitreached] = true end if retry @debugv 1 "🔄 Retry $ex: $(sprintcompact(req))" reset!(req.response) if req_body_is_marked @debugv 2 "Resetting request body stream" reset(req.body) mark(req.body) end else @debugv 1 "🚷 No Retry: $(no_retry_reason(ex, req))" end return s, retry end ) return retry_request(req; kw...) end end isrecoverable(ex) = is_wrapped_exception(ex) ? isrecoverable(unwrap_exception(ex)) : false isrecoverable(::Union{Base.EOFError, Base.IOError, MbedTLS.MbedException, OpenSSL.OpenSSLError}) = true isrecoverable(ex::ArgumentError) = ex.msg == "stream is closed or unusable" isrecoverable(ex::CompositeException) = all(isrecoverable, ex.exceptions) # Treat all DNS errors except `EAI_AGAIN`` as non-recoverable # Ref: https://github.com/JuliaLang/julia/blob/ec8df3da3597d0acd503ff85ac84a5f8f73f625b/stdlib/Sockets/src/addrinfo.jl#L108-L112 isrecoverable(ex::Sockets.DNSError) = (ex.code == Base.UV_EAI_AGAIN) isrecoverable(ex::StatusError) = retryable(ex.status) function _retry_check(s, ex, req, check) resp = req.response resp_body = get(req.context, :response_body, nothing) return check(s, ex, req, resp_body !== nothing ? resp : nothing, resp_body) end function no_retry_reason(ex, req) buf = IOBuffer() unwrapped_ex = unwrap_exception(ex) show(IOContext(buf, :compact => true), req) print(buf, ", ", unwrapped_ex isa StatusError ? "HTTP $(ex.status): " : !isrecoverable(unwrapped_ex) ? "unrecoverable exception: " : !isbytes(req.body) ? "request streamed, " : "", !isbytes(req.response.body) ? "response streamed, " : "", !isidempotent(req) ? "$(req.method) non-idempotent" : "") return String(take!(buf)) end end # module RetryRequest D/opt/julia/packages/HTTP/sJD5V/src/clientlayers/ConnectionRequest.jl$module ConnectionRequest using URIs, Sockets, Base64, LoggingExtras, ConcurrentUtilities, ExceptionUnwrapping import MbedTLS import OpenSSL using ..Messages, ..IOExtras, ..Connections, ..Streams, ..Exceptions import ..SOCKET_TYPE_TLS islocalhost(host::AbstractString) = host == "localhost" || host == "127.0.0.1" || host == "::1" || host == "0000:0000:0000:0000:0000:0000:0000:0001" || host == "0:0:0:0:0:0:0:1" # hasdotsuffix reports whether s ends in "."+suffix. hasdotsuffix(s, suffix) = endswith(s, "." * suffix) function isnoproxy(host::AbstractString) for x in NO_PROXY (hasdotsuffix(host, x) || (host == x)) && return true end return false end const NO_PROXY = String[] function __init__() # check for no_proxy environment variable if haskey(ENV, "no_proxy") for x in split(ENV["no_proxy"], ","; keepempty=false) push!(NO_PROXY, startswith(x, ".") ? x[2:end] : x) end end return end function getproxy(scheme, host) (isnoproxy(host) || islocalhost(host)) && return nothing if scheme == "http" && (p = get(ENV, "http_proxy", ""); !isempty(p)) return p elseif scheme == "http" && (p = get(ENV, "HTTP_PROXY", ""); !isempty(p)) return p elseif scheme == "https" && (p = get(ENV, "https_proxy", ""); !isempty(p)) return p elseif scheme == "https" && (p = get(ENV, "HTTPS_PROXY", ""); !isempty(p)) return p end return nothing end export connectionlayer const CLOSE_IMMEDIATELY = Ref{Bool}(false) """ connectionlayer(handler) -> handler Retrieve an `IO` connection from the Connections. Close the connection if the request throws an exception. Otherwise leave it open so that it can be reused. """ function connectionlayer(handler) return function connections(req; proxy=getproxy(req.url.scheme, req.url.host), socket_type::Type=TCPSocket, socket_type_tls::Union{Nothing, Type}=nothing, readtimeout::Int=0, connect_timeout::Int=30, logerrors::Bool=false, logtag=nothing, closeimmediately::Bool=CLOSE_IMMEDIATELY[], kw...) local io, stream if proxy !== nothing target_url = req.url url = URI(proxy) if target_url.scheme == "http" req.target = string(target_url) end userinfo = unescapeuri(url.userinfo) if !isempty(userinfo) && !hasheader(req.headers, "Proxy-Authorization") @debugv 1 "Adding Proxy-Authorization: Basic header." setheader(req.headers, "Proxy-Authorization" => "Basic $(base64encode(userinfo))") end else url = target_url = req.url end connect_timeout = connect_timeout == 0 && readtimeout > 0 ? readtimeout : connect_timeout IOType = sockettype(url, socket_type, socket_type_tls, get(kw, :sslconfig, nothing)) start_time = time() try io = newconnection(IOType, url.host, url.port; readtimeout=readtimeout, connect_timeout=connect_timeout, kw...) catch e if logerrors @error current_exceptions_to_string() type=Symbol("HTTP.ConnectError") method=req.method url=req.url context=req.context logtag=logtag end req.context[:connect_errors] = get(req.context, :connect_errors, 0) + 1 throw(ConnectError(string(url), e)) finally req.context[:connect_duration_ms] = get(req.context, :connect_duration_ms, 0.0) + (time() - start_time) * 1000 end shouldreuse = !(target_url.scheme in ("ws", "wss")) && !closeimmediately try if proxy !== nothing && target_url.scheme in ("https", "wss", "ws") shouldreuse = false # tunnel request if target_url.scheme in ("https", "wss") target_url = URI(target_url, port=443) elseif target_url.scheme in ("ws", ) && target_url.port == "" target_url = URI(target_url, port=80) # if there is no port info, connect_tunnel will fail end r = if readtimeout > 0 try_with_timeout(readtimeout) do _ connect_tunnel(io, target_url, req) end else connect_tunnel(io, target_url, req) end if r.status != 200 close(io) return r end if target_url.scheme in ("https", "wss") InnerIOType = sockettype(target_url, socket_type, socket_type_tls, get(kw, :sslconfig, nothing)) io = Connections.sslupgrade(InnerIOType, io, target_url.host; readtimeout=readtimeout, kw...) end req.headers = filter(x->x.first != "Proxy-Authorization", req.headers) end stream = Stream(req.response, io) return handler(stream; readtimeout=readtimeout, logerrors=logerrors, logtag=logtag, kw...) catch e shouldreuse = false # manually unwrap CompositeException since it's not defined as a "wrapper" exception by ExceptionUnwrapping while e isa CompositeException e = e.exceptions[1] end root_err = ExceptionUnwrapping.unwrap_exception_to_root(e) # don't log if it's an HTTPError since we should have already logged it if logerrors && root_err isa StatusError @error current_exceptions_to_string() type=Symbol("HTTP.StatusError") method=req.method url=req.url context=req.context logtag=logtag end if logerrors && !ExceptionUnwrapping.has_wrapped_exception(e, HTTPError) @error current_exceptions_to_string() type=Symbol("HTTP.ConnectionRequest") method=req.method url=req.url context=req.context logtag=logtag end @debugv 1 "❗️ ConnectionLayer $root_err. Closing: $io" if @isdefined(stream) && stream.nwritten == -1 # we didn't write anything, so don't need to worry about # idempotency of the request req.context[:nothingwritten] = true end root_err isa HTTPError || throw(RequestError(req, root_err)) throw(root_err) finally releaseconnection(io, shouldreuse; kw...) if !shouldreuse @try Base.IOError close(io) end end end end function sockettype(url::URI, socket_type_tcp, socket_type_tls, sslconfig) if url.scheme in ("wss", "https") tls_socket_type(socket_type_tls, sslconfig) else socket_type_tcp end end """ tls_socket_type(socket_type_tls, sslconfig)::Type Find the best TLS socket type, given the values of these keyword arguments. If both are `nothing` then we use the global default: `HTTP.SOCKET_TYPE_TLS[]`. If both are not `nothing` then they must agree: `sslconfig` must be of the right type to configure `socket_type_tls` or we throw an `ArgumentError`. """ function tls_socket_type(socket_type_tls::Union{Nothing, Type}, sslconfig::Union{Nothing, MbedTLS.SSLConfig, OpenSSL.SSLContext} )::Type socket_type_matching_sslconfig = if sslconfig isa MbedTLS.SSLConfig MbedTLS.SSLContext elseif sslconfig isa OpenSSL.SSLContext OpenSSL.SSLStream else nothing end if socket_type_tls === socket_type_matching_sslconfig # Use the global default TLS socket if they're both nothing, or use # what they both specify if they're not nothing. isnothing(socket_type_tls) ? SOCKET_TYPE_TLS[] : socket_type_tls # If either is nothing, use the other one. elseif isnothing(socket_type_tls) socket_type_matching_sslconfig elseif isnothing(socket_type_matching_sslconfig) socket_type_tls else # If they specify contradictory types, throw an error. # Error thrown in noinline closure to avoid speed penalty in common case @noinline function err(socket_type_tls, sslconfig) msg = """ Incompatible values for keyword args `socket_type_tls` and `sslconfig`: socket_type_tls=$socket_type_tls typeof(sslconfig)=$(typeof(sslconfig)) Make them match or provide only one of them. - the socket type MbedTLS.SSLContext is configured by MbedTLS.SSLConfig - the socket type OpenSSL.SSLStream is configured by OpenSSL.SSLContext""" throw(ArgumentError(msg)) end err(socket_type_tls, sslconfig) end end function connect_tunnel(io, target_url, req) target = "$(URIs.hoststring(target_url.host)):$(target_url.port)" @debugv 1 "📡 CONNECT HTTPS tunnel to $target" headers = Dict("Host" => target) if (auth = header(req, "Proxy-Authorization"); !isempty(auth)) headers["Proxy-Authorization"] = auth end request = Request("CONNECT", target, headers) # @debugv 2 "connect_tunnel: writing headers" writeheaders(io, request) # @debugv 2 "connect_tunnel: reading headers" readheaders(io, request.response) # @debugv 2 "connect_tunnel: done reading headers" return request.response end end # module ConnectionRequest @/opt/julia/packages/HTTP/sJD5V/src/clientlayers/StreamRequest.jl@module StreamRequest using ..IOExtras, ..Messages, ..Streams, ..Connections, ..Strings, ..RedirectRequest, ..Exceptions using LoggingExtras, CodecZlib, URIs using SimpleBufferStream: BufferStream using ConcurrentUtilities: @samethreadpool_spawn export streamlayer """ streamlayer(stream) -> HTTP.Response Create a [`Stream`](@ref) to send a `Request` and `body` to an `IO` stream and read the response. Send the `Request` body in a background task and begins reading the response immediately so that the transmission can be aborted if the `Response` status indicates that the server does not wish to receive the message body. [RFC7230 6.5](https://tools.ietf.org/html/rfc7230#section-6.5). """ function streamlayer(stream::Stream; iofunction=nothing, decompress::Union{Nothing, Bool}=nothing, logerrors::Bool=false, logtag=nothing, timedout=nothing, kw...)::Response response = stream.message req = response.request @debugv 1 sprintcompact(req) @debugv 2 "client startwrite" write_start = time() startwrite(stream) @debugv 2 sprint(show, req) if iofunction === nothing && !isbytes(req.body) @debugv 2 "$(typeof(req)).body: $(sprintcompact(req.body))" end try @sync begin if iofunction === nothing # use a lock here for request.context changes (this is currently the only places # where multiple threads may modify/change context at the same time) lock = ReentrantLock() @samethreadpool_spawn try writebody(stream, req, lock) finally Base.@lock lock begin req.context[:write_duration_ms] = get(req.context, :write_duration_ms, 0.0) + ((time() - write_start) * 1000) end @debugv 2 "client closewrite" closewrite(stream) end read_start = time() @samethreadpool_spawn try @debugv 2 "client startread" startread(stream) if !isaborted(stream) readbody(stream, response, decompress, lock) end finally Base.@lock lock begin req.context[:read_duration_ms] = get(req.context, :read_duration_ms, 0.0) + ((time() - read_start) * 1000) end @debugv 2 "client closeread" closeread(stream) end else try iofunction(stream) finally closewrite(stream) closeread(stream) end end end catch if timedout === nothing || !timedout[] req.context[:io_errors] = get(req.context, :io_errors, 0) + 1 if logerrors @error current_exceptions_to_string() type=Symbol("HTTP.IOError") method=req.method url=req.url context=req.context logtag=logtag end end rethrow() end @debugv 1 sprintcompact(response) @debugv 2 sprint(show, response) return response end function writebody(stream::Stream, req::Request, lock) if !isbytes(req.body) n = writebodystream(stream, req.body) closebody(stream) else n = write(stream, req.body) end Base.@lock lock begin req.context[:nbytes_written] = n end return n end function writebodystream(stream, body) n = 0 for chunk in body n += writechunk(stream, chunk) end return n end function writebodystream(stream, body::IO) return write(stream, body) end function writebodystream(stream, body::Union{AbstractDict, NamedTuple}) # application/x-www-form-urlencoded return write(stream, URIs.escapeuri(body)) end writechunk(stream, body::IO) = writebodystream(stream, body) writechunk(stream, body::Union{AbstractDict, NamedTuple}) = writebodystream(stream, body) writechunk(stream, body) = write(stream, body) function readbody(stream::Stream, res::Response, decompress::Union{Nothing, Bool}, lock) if decompress === true || (decompress === nothing && header(res, "Content-Encoding") == "gzip") # Plug in a buffer stream in between so that we can (i) read the http stream in # chunks instead of byte-by-byte and (ii) make sure to stop reading the http stream # at eof. buf = BufferStream() gzstream = GzipDecompressorStream(buf) tsk = @async begin try write(gzstream, stream) finally # Close here to (i) deallocate resources in zlib and (ii) make sure that # read(buf)/write(..., buf) below don't block forever. Note that this will # close the stream wrapped by the decompressor (buf) but *not* the http # stream, which should be left open. close(gzstream) end end readbody!(stream, res, buf, lock) wait(tsk) else readbody!(stream, res, stream, lock) end end function readbody!(stream::Stream, res::Response, buf_or_stream, lock) n = 0 if !iserror(res) if isbytes(res.body) if length(res.body) > 0 # user-provided buffer to read response body into # specify write=true to make the buffer writable # but also specify maxsize, which means it won't be grown # (we don't want to be changing the user's buffer for them) body = IOBuffer(res.body; write=true, maxsize=length(res.body)) if buf_or_stream isa BufferStream # if it's a BufferStream, the response body was gzip encoded # so using the default write is fastest because it utilizes # readavailable under the hood, for which BufferStream is optimized n = write(body, buf_or_stream) elseif buf_or_stream isa Stream{Response} # for HTTP.Stream, there's already an optimized read method # that just needs an IOBuffer to write into n = readall!(buf_or_stream, body) else error("unreachable") end else res.body = read(buf_or_stream) n = length(res.body) end elseif res.body isa Base.GenericIOBuffer && buf_or_stream isa Stream{Response} # optimization for IOBuffer response_stream to avoid temporary allocations n = readall!(buf_or_stream, res.body) else n = write(res.body, buf_or_stream) end else # read the response body into the request context so that it can be # read by the user if they want to or set later if # we end up not retrying/redirecting/etc. Base.@lock lock begin res.request.context[:response_body] = read(buf_or_stream) end end Base.@lock lock begin res.request.context[:nbytes] = n end end end # module StreamRequest ./opt/julia/packages/HTTP/sJD5V/src/download.jlusing .Pairs using CodecZlib """ safer_joinpath(basepart, parts...) A variation on `joinpath`, that is more resistant to directory traversal attacks. The parts to be joined (excluding the `basepart`), are not allowed to contain `..`, or begin with a `/`. If they do then this throws an `DomainError`. """ function safer_joinpath(basepart, parts...) explain = "Possible directory traversal attack detected." for part in parts occursin("..", part) && throw(DomainError(part, "contains \"..\". $explain")) startswith(part, '/') && throw(DomainError(part, "begins with \"/\". $explain")) end joinpath(basepart, parts...) end function try_get_filename_from_headers(hdrs) for content_disp in hdrs # extract out of Content-Disposition line # rough version of what is needed in https://github.com/JuliaWeb/HTTP.jl/issues/179 filename_part = match(r"filename\s*=\s*(.*)", content_disp) if filename_part !== nothing filename = filename_part[1] quoted_filename = match(r"\"(.*)\"", filename) if quoted_filename !== nothing # It was in quotes, so it will be double escaped filename = unescape_string(quoted_filename[1]) end return filename == "" ? nothing : filename end end return nothing end function try_get_filename_from_request(req) function file_from_target(t) (t == "" || t == "/") && return nothing f = basename(URI(t).path) # URI(...).path to strip out e.g. query parts return (f == "" ? file_from_target(dirname(t)) : f) end # First try to get file from the original request URI oreq = req while oreq.parent !== nothing oreq = oreq.parent.request end f = file_from_target(oreq.target) f !== nothing && return f # Secondly try to get file from the last request URI return file_from_target(req.target) end determine_file(::Nothing, resp, hdrs) = determine_file(tempdir(), resp, hdrs) # ^ We want to the filename if possible because extension is useful for FileIO.jl function determine_file(path, resp, hdrs) if isdir(path) # we have been given a path to a directory # got to to workout what file to put there filename = something( try_get_filename_from_headers(hdrs), try_get_filename_from_request(resp.request), basename(tempname()) # fallback, basically a random string ) safer_joinpath(path, filename) else # We have been given a full filepath path end end """ download(url, [local_path], [headers]; update_period=1, kw...) Similar to `Base.download` this downloads a file, returning the filename. If the `local_path`: - is not provided, then it is saved in a temporary directory - if part to a directory is provided then it is saved into that directory - otherwise the local path is uses as the filename to save to. When saving into a directory, the filename is determined (where possible), from the rules of the HTTP. - `update_period` controls how often (in seconds) to report the progress. - set to `Inf` to disable reporting - `headers` specifies headers to be used for the HTTP GET request - any additional keyword args (`kw...`) are passed on to the HTTP request. """ function download(url::AbstractString, local_path=nothing, headers=Header[]; update_period=1, kw...) format_progress(x) = round(x, digits=4) format_bytes(x) = !isfinite(x) ? "∞ B" : Base.format_bytes(round(Int, x)) format_seconds(x) = "$(round(x; digits=2)) s" format_bytes_per_second(x) = format_bytes(x) * "/s" @debugv 1 "downloading $url" local file hdrs = String[] HTTP.open("GET", url, headers; kw...) do stream resp = startread(stream) # Store intermediate header from redirects to use for filename detection content_disp = header(resp, "Content-Disposition") !isempty(content_disp) && push!(hdrs, content_disp) eof(stream) && return # don't do anything for streams we can't read (yet) file = determine_file(local_path, resp, hdrs) total_bytes = parse(Float64, header(resp, "Content-Length", "NaN")) downloaded_bytes = 0 start_time = now() prev_time = now() if header(resp, "Content-Encoding") == "gzip" stream = GzipDecompressorStream(stream) # auto decoding total_bytes = NaN # We don't know actual total bytes if the content is zipped. end function report_callback() prev_time = now() taken_time = (prev_time - start_time).value / 1000 # in seconds average_speed = downloaded_bytes / taken_time remaining_bytes = total_bytes - downloaded_bytes remaining_time = remaining_bytes / average_speed completion_progress = downloaded_bytes / total_bytes @info("Downloading", source=url, dest = file, progress = completion_progress |> format_progress, time_taken = taken_time |> format_seconds, time_remaining = remaining_time |> format_seconds, average_speed = average_speed |> format_bytes_per_second, downloaded = downloaded_bytes |> format_bytes, remaining = remaining_bytes |> format_bytes, total = total_bytes |> format_bytes, ) end Base.open(file, "w") do fh while(!eof(stream)) downloaded_bytes += write(fh, readavailable(stream)) if !isinf(update_period) if now() - prev_time > Millisecond(round(1000update_period)) report_callback() end end end end if !isinf(update_period) report_callback() end end file end -/opt/julia/packages/HTTP/sJD5V/src/Servers.jlSK""" The `HTTP.Servers` module provides core HTTP server functionality. The main entry point is `HTTP.listen(f, host, port; kw...)` which takes a `f(::HTTP.Stream)::Nothing` function argument, a `host`, a `port` and optional keyword arguments. For full details, see `?HTTP.listen`. For server functionality operating on full requests, see the `?HTTP.serve` function. """ module Servers export listen, listen!, Server, forceclose, port using Sockets, Logging, LoggingExtras, MbedTLS, Dates using MbedTLS: SSLContext, SSLConfig using ..IOExtras, ..Streams, ..Messages, ..Parsers, ..Connections, ..Exceptions import ..access_threaded, ..SOCKET_TYPE_TLS, ..@logfmt_str TRUE(x) = true getinet(host::String, port::Integer) = Sockets.InetAddr(parse(IPAddr, host), port) getinet(host::IPAddr, port::Integer) = Sockets.InetAddr(host, port) struct Listener{S <: Union{SSLConfig, Nothing}, I <: Base.IOServer} addr::Sockets.InetAddr hostname::String hostport::String ssl::S server::I end function Listener(server::Base.IOServer; sslconfig::Union{MbedTLS.SSLConfig, Nothing}=nothing, kw...) host, port = getsockname(server) addr = getinet(host, port) return Listener(addr, string(host), string(port), sslconfig, server) end supportsreuseaddr() = ccall(:jl_has_so_reuseport, Int32, ()) == 1 function Listener(addr::Sockets.InetAddr, host::String, port::String; sslconfig::Union{MbedTLS.SSLConfig, Nothing}=nothing, reuseaddr::Bool=false, backlog::Integer=Sockets.BACKLOG_DEFAULT, server::Union{Nothing, Base.IOServer}=nothing, # for backwards compat listenany::Bool=false, kw...) if server !== nothing return Listener(server; sslconfig=sslconfig) end if listenany p, server = Sockets.listenany(addr.host, addr.port) addr = getinet(addr.host, p) port = string(p) elseif reuseaddr if !supportsreuseaddr() @warn "reuseaddr=true not supported on this platform: $(Sys.KERNEL)" @goto fallback end server = Sockets.TCPServer(delay = false) rc = ccall(:jl_tcp_reuseport, Int32, (Ptr{Cvoid},), server.handle) if rc < 0 close(server) @warn "reuseaddr=true failed; falling back to regular listen: $(Sys.KERNEL)" @goto fallback end Sockets.bind(server, addr.host, addr.port; reuseaddr=true) Sockets.listen(server; backlog=backlog) else @label fallback server = Sockets.listen(addr; backlog=backlog) end return Listener(addr, host, port, sslconfig, server) end Listener(host::Union{IPAddr, String}, port::Integer; kw...) = Listener(getinet(host, port), string(host), string(port); kw...) Listener(port::Integer; kw...) = Listener(Sockets.localhost, port; kw...) Listener(; kw...) = Listener(Sockets.localhost, 8081; kw...) Base.isopen(l::Listener) = isopen(l.server) Base.close(l::Listener) = close(l.server) accept(s::Listener{Nothing}) = Sockets.accept(s.server)::TCPSocket accept(s::Listener{SSLConfig}) = getsslcontext(Sockets.accept(s.server), s.ssl) function getsslcontext(tcp, sslconfig) try ssl = MbedTLS.SSLContext() MbedTLS.setup!(ssl, sslconfig) MbedTLS.associate!(ssl, tcp) MbedTLS.handshake!(ssl) return ssl catch e @try Base.IOError close(tcp) e isa Base.IOError && return nothing e isa MbedTLS.MbedException && return nothing rethrow(e) end end """ HTTP.Server Returned from `HTTP.listen!`/`HTTP.serve!` once a server is up listening and ready to accept connections. Internally keeps track of active connections. Also holds reference to any `on_shutdown` functions to be called when the server is closed. Also holds a reference to the listening loop `Task` that can be waited on via `wait(server)`, which provides similar functionality to `HTTP.listen`/ `HTTP.serve`. Can initiate a graceful shutdown where active connections are allowed to finish being handled by calling `close(server)`. For a more forceful and immediate shutdown, use `HTTP.forceclose(server)`. """ struct Server{L <: Listener} # listener socket + details listener::L # optional function or vector of functions # to call when closing server on_shutdown::Any # list of currently acctive connections connections::Set{Connection} # server listenandserve loop task task::Task # Protects the connections Set which is mutated in the listenloop # while potentially being accessed by the close method at the same time connections_lock::ReentrantLock end port(s::Server) = Int(s.listener.addr.port) Base.isopen(s::Server) = isopen(s.listener) Base.wait(s::Server) = wait(s.task) function forceclose(s::Server) shutdown(s.on_shutdown) close(s.listener) Base.@lock s.connections_lock begin for c in s.connections close(c) end end return wait(s.task) end """ ConnectionState When a connection is first made, it immediately goes into IDLE state. Once startread(stream) returns, it's marked as ACTIVE. Once closewrite(stream) returns, it's put back in IDLE, unless it's been given the CLOSING state, then it will close(c) itself and mark itself as CLOSED. """ @enum ConnectionState IDLE ACTIVE CLOSING CLOSED closedorclosing(st) = st == CLOSING || st == CLOSED function requestclose!(c::Connection) if c.state == IDLE c.state = CLOSED close(c) else c.state = CLOSING end return end function closeconnection(c::Connection) c.state = CLOSED close(c) return end # graceful shutdown that waits for active connectiosn to finish being handled function Base.close(s::Server) shutdown(s.on_shutdown) close(s.listener) # first pass to mark or request connections to close Base.@lock s.connections_lock begin for c in s.connections requestclose!(c) end end # second pass to wait for connections to close # we wait for connections to empty because as # connections close themselves, they are removed # from our connections Set while true Base.@lock s.connections_lock begin isempty(s.connections) && break end sleep(0.5 + rand() * 0.1) end return wait(s.task) end """ shutdown(fns::Vector{<:Function}) shutdown(fn::Function) shutdown(::Nothing) Runs function(s) in `on_shutdown` field of `Server` when `Server` is closed. """ shutdown(fns::Vector{<:Function}) = foreach(shutdown, fns) shutdown(::Nothing) = nothing function shutdown(fn::Function) try fn() catch @error begin msg = current_exceptions_to_string() "shutdown function $fn failed. $msg" end end end """ HTTP.listen(handler, host=Sockets.localhost, port=8081; kw...) HTTP.listen(handler, port::Integer=8081; kw...) HTTP.listen(handler, server::Base.IOServer; kw...) HTTP.listen!(args...; kw...) -> HTTP.Server Listen for HTTP connections and execute the `handler` function for each request. Listening details can be passed as `host`/`port` pair, a single `port` (`host` will default to `localhost`), or an already listening `server` object, as returned from `Sockets.listen`. To open up a server to external requests, the `host` argument is typically `"0.0.0.0"`. The `HTTP.listen!` form is non-blocking and returns an `HTTP.Server` object which can be `wait(server)`ed on manually, or `close(server)`ed to gracefully shut down the server. Calling `HTTP.forceclose(server)` will immediately force close the server and all active connections. `HTTP.listen` will block on the server listening loop until interrupted or and an irrecoverable error occurs. The `handler` function should be of the form `f(::HTTP.Stream)::Nothing`, and should at the minimum set a status via `setstatus()` and call `startwrite()` either explicitly or implicitly by writing out a response via `write()`. Failure to do this will result in an HTTP 500 error being transmitted to the client. Optional keyword arguments: - `sslconfig=nothing`, Provide an `MbedTLS.SSLConfig` object to handle ssl connections. Pass `sslconfig=MbedTLS.SSLConfig(false)` to disable ssl verification (useful for testing). Construct a custom `SSLConfig` object with `MbedTLS.SSLConfig(certfile, keyfile)`. - `tcpisvalid = tcp->true`, function `f(::TCPSocket)::Bool` to check if accepted connections are valid before processing requests. e.g. to do source IP filtering. - `readtimeout::Int=0`, close the connection if no data is received for this many seconds. Use readtimeout = 0 to disable. - `reuseaddr::Bool=false`, allow multiple servers to listen on the same port. Not supported on some OS platforms. Can check `HTTP.Servers.supportsreuseaddr()`. - `server::Base.IOServer=nothing`, provide an `IOServer` object to listen on; allows manually closing or configuring the server socket. - `verbose::Bool=false`, log connection information to `stdout`. - `access_log::Function`, function for formatting access log messages. The function should accept two arguments, `io::IO` to which the messages should be written, and `http::HTTP.Stream` which can be used to query information from. See also [`@logfmt_str`](@ref). - `on_shutdown::Union{Function, Vector{<:Function}, Nothing}=nothing`, one or more functions to be run if the server is closed (for example by an `InterruptException`). Note, shutdown function(s) will not run if an `IOServer` object is supplied to the `server` keyword argument and closed by `close(server)`. e.g. ```julia # start a blocking server HTTP.listen("127.0.0.1", 8081) do http HTTP.setheader(http, "Content-Type" => "text/html") write(http, "target uri: \$(http.message.target)
") write(http, "request body:
")
    write(http, read(http))
    write(http, "
") return end # non-blocking server server = HTTP.listen!("127.0.0.1", 8081) do http @show http.message @show HTTP.header(http, "Content-Type") while !eof(http) println("body data: ", String(readavailable(http))) end HTTP.setstatus(http, 404) HTTP.setheader(http, "Foo-Header" => "bar") startwrite(http) write(http, "response body") write(http, "more response body") end # can gracefully close server manually close(server) ``` To run the following HTTP chat example, open two Julia REPL windows and paste the example code into both of them. Then in one window run `chat_server()` and in the other run `chat_client()`, then type `hello` and press return. Whatever you type on the client will be displayed on the server and vis-versa. ``` using HTTP function chat(io::HTTP.Stream) Threads.@spawn while !eof(io) write(stdout, readavailable(io), "\\n") end while isopen(io) write(io, readline(stdin)) end end chat_server() = HTTP.listen("127.0.0.1", 8087) do io write(io, "HTTP.jl Chat Server. Welcome!") chat(io) end chat_client() = HTTP.open("POST", "http://127.0.0.1:8087") do io chat(io) end ``` """ function listen end """ HTTP.listen!(args...; kw...) -> HTTP.Server Non-blocking version of [`HTTP.listen`](@ref); see that function for details. """ function listen! end listen(f, args...; kw...) = listen(f, Listener(args...; kw...); kw...) listen!(f, args...; kw...) = listen!(f, Listener(args...; kw...); kw...) function listen(f, listener::Listener; kw...) server = listen!(f, listener; kw...) # block on server task try wait(server) finally # try to gracefully close close(server) end return server end # compat for `Threads.@spawn :interactive expr` @static if hasmethod(getfield(Threads, Symbol("@spawn")), Tuple{LineNumberNode, Module, Symbol, Expr}) macro _spawn_interactive(ex) esc(:(Threads.@spawn :interactive $ex)) end else macro _spawn_interactive(ex) esc(:(@async $ex)) end end function listen!(f, listener::Listener; on_shutdown=nothing, tcpisvalid=TRUE, max_connections::Integer=typemax(Int), readtimeout::Integer=0, access_log::Union{Function,Nothing}=nothing, verbose=false, kw...) conns = Set{Connection}() conns_lock = ReentrantLock() ready_to_accept = Threads.Event() if verbose > 0 tsk = @_spawn_interactive LoggingExtras.withlevel(Logging.Debug; verbosity=verbose) do listenloop(f, listener, conns, tcpisvalid, max_connections, readtimeout, access_log, ready_to_accept, conns_lock, verbose) end else tsk = @_spawn_interactive listenloop(f, listener, conns, tcpisvalid, max_connections, readtimeout, access_log, ready_to_accept, conns_lock, verbose) end # wait until the listenloop enters the loop wait(ready_to_accept) return Server(listener, on_shutdown, conns, tsk, conns_lock) end """" Main server loop. Accepts new tcp connections and spawns async tasks to handle them." """ function listenloop( f, listener, conns, tcpisvalid, max_connections, readtimeout, access_log, ready_to_accept, conns_lock, verbose ) sem = Base.Semaphore(max_connections) verbose >= 0 && @infov 1 "Listening on: $(listener.hostname):$(listener.hostport), thread id: $(Threads.threadid())" notify(ready_to_accept) while isopen(listener) try Base.acquire(sem) io = accept(listener) if io === nothing @warnv 1 "unable to accept new connection" continue elseif !tcpisvalid(io) @warnv 1 "!tcpisvalid: $io" close(io) continue end conn = Connection(io) conn.state = IDLE Base.@lock conns_lock push!(conns, conn) conn.host, conn.port = listener.hostname, listener.hostport @async try handle_connection(f, conn, listener, readtimeout, access_log) finally # handle_connection is in charge of closing the underlying io Base.@lock conns_lock delete!(conns, conn) Base.release(sem) end catch e if e isa Base.IOError && e.code == Base.UV_ECONNABORTED verbose >= 0 && @infov 1 "Server on $(listener.hostname):$(listener.hostport) closing" else @errorv 2 begin msg = current_exceptions_to_string() "Server on $(listener.hostname):$(listener.hostport) errored. $msg" end # quick little sleep in case there's a temporary # local error accepting and this might help avoid quickly re-erroring sleep(0.05 + rand() * 0.05) end end end return end """ Start a `check_readtimeout` task to close the `Connection` if it is inactive. Passes the `Connection` object to handle a single request/response transaction for each HTTP Request received. After `reuse_limit + 1` transactions, signal `final_transaction` to the transaction handler, which will close the connection. """ function handle_connection(f, c::Connection, listener, readtimeout, access_log) wait_for_timeout = Ref{Bool}(true) if readtimeout > 0 @async check_readtimeout(c, readtimeout, wait_for_timeout) end try # if the connection socket or listener close, we stop taking requests while isopen(c) && !closedorclosing(c.state) && isopen(listener) # create new Request to be populated by parsing code request = Request() # wrap Request in Stream w/ Connection for request reading/response writing http = Stream(request, c) # attempt to read request line and headers try startread(http) @debugv 1 "startread called" c.state = ACTIVE # once we've started reading, set ACTIVE state catch e # for ParserErrors, try to inform client of the problem if e isa ParseError write(c, Response(e.code == :HEADER_SIZE_EXCEEDS_LIMIT ? 431 : 400, string(e.code))) end @debugv 1 begin msg = current_exceptions_to_string() "handle_connection startread error. $msg" end break end if hasheader(request, "Connection", "close") c.state = CLOSING # set CLOSING so no more requests are read setheader(request.response, "Connection" => "close") end request.response.status = 200 try # invokelatest becuase the perf is negligible, but this makes live-editing handlers more Revise friendly @debugv 1 "invoking handler" Base.invokelatest(f, http) # If `startwrite()` was never called, throw an error so we send a 500 and log this if isopen(http) && !iswritable(http) error("Server never wrote a response.\n\n$request") end @debugv 1 "closeread" closeread(http) @debugv 1 "closewrite" closewrite(http) c.state = IDLE catch e # The remote can close the stream whenever it wants to, but there's nothing # anyone can do about it on this side. No reason to log an error in that case. level = e isa Base.IOError && !isopen(c) ? Logging.Debug : Logging.Error @logmsgv 1 level begin msg = current_exceptions_to_string() "handle_connection handler error. $msg" end request if isopen(http) && !iswritable(http) request.response.status = 500 startwrite(http) closewrite(http) end c.state = CLOSING finally if access_log !== nothing @try(Any, @info sprint(access_log, http) _group=:access) end end end catch # we should be catching everything inside the while loop, but just in case @errorv 1 begin msg = current_exceptions_to_string() "error while handling connection. $msg" end finally if readtimeout > 0 wait_for_timeout[] = false end # when we're done w/ the connection, ensure it's closed and state is properly set closeconnection(c) end return end """ If `c` is inactive for a more than `readtimeout` then close the `c`." """ function check_readtimeout(c, readtimeout, wait_for_timeout) while wait_for_timeout[] if inactiveseconds(c) > readtimeout @warnv 2 "Connection Timeout: $c" try writeheaders(c, Response(408, ["Connection" => "close"])) finally closeconnection(c) end break end sleep(readtimeout + rand() * readtimeout) end return end end # module ./opt/julia/packages/HTTP/sJD5V/src/Handlers.jlJmodule Handlers export Handler, Middleware, serve, serve!, Router, register!, getroute, getparams, getparam, getcookies, streamhandler using URIs using ..Messages, ..Streams, ..IOExtras, ..Servers, ..Sockets, ..Cookies import ..HTTP # for doc references """ Handler Abstract type for the handler interface that exists for documentation purposes. A `Handler` is any function of the form `f(req::HTTP.Request) -> HTTP.Response`. There is no requirement to subtype `Handler` and users should not rely on or dispatch on `Handler`. A `Handler` function `f` can be passed to [`HTTP.serve`](@ref) wherein a server will pass each incoming request to `f` to be handled and a response to be returned. Handler functions are also the inputs to [`Middleware`](@ref) functions which are functions of the form `f(::Handler) -> Handler`, i.e. they take a `Handler` function as input, and return a "modified" or enhanced `Handler` function. For advanced cases, a `Handler` function can also be of the form `f(stream::HTTP.Stream) -> Nothing`. In this case, the server would be run like `HTTP.serve(f, ...; stream=true)`. For this use-case, the handler function reads the request and writes the response to the stream directly. Note that any middleware used with a stream handler also needs to be of the form `f(stream_handler) -> stream_handler`, i.e. it needs to accept a stream `Handler` function and return a stream `Handler` function. """ abstract type Handler end """ Middleware Abstract type for the middleware interface that exists for documentation purposes. A `Middleware` is any function of the form `f(::Handler) -> Handler` (ref: [`Handler`](@ref)). There is no requirement to subtype `Middleware` and users should not rely on or dispatch on the `Middleware` type. While `HTTP.serve(f, ...)` requires a _handler_ function `f` to be passed, middleware can be "stacked" to create a chain of functions that are called in sequence, like `HTTP.serve(base_handler |> cookie_middleware |> auth_middlware, ...)`, where the `base_handler` `Handler` function is passed to `cookie_middleware`, which takes the handler and returns a "modified" handler (that parses and stores cookies). This "modified" handler is then an input to the `auth_middlware`, which further enhances/modifies the handler. """ abstract type Middleware end """ streamhandler(request_handler) -> stream handler Middleware that takes a request handler and returns a stream handler. Used by default in `HTTP.serve` to take the user-provided request handler and process the `Stream` from `HTTP.listen` and pass the parsed `Request` to the handler. Is included by default in `HTTP.serve` as the base "middleware" when `stream=false` is passed. """ function streamhandler(handler) return function(stream::Stream) request::Request = stream.message request.body = read(stream) closeread(stream) request.response::Response = handler(request) request.response.request = request startwrite(stream) write(stream, request.response.body) return end end # Interface change in HTTP@1 @deprecate RequestHandlerFunction streamhandler """ HTTP.serve(handler, host=Sockets.localhost, port=8081; kw...) HTTP.serve(handler, port::Integer=8081; kw...) HTTP.serve(handler, server::Base.IOServer; kw...) HTTP.serve!(args...; kw...) -> HTTP.Server Listen for HTTP connections and execute the `handler` function for each request. Listening details can be passed as `host`/`port` pair, a single `port` (`host` will default to `localhost`), or an already listening `server` object, as returned from `Sockets.listen`. To open up a server to external requests, the `host` argument is typically `"0.0.0.0"`. The `HTTP.serve!` form is non-blocking and returns an `HTTP.Server` object which can be `wait(server)`ed on manually, or `close(server)`ed to gracefully shut down the server. Calling `HTTP.forceclose(server)` will immediately force close the server and all active connections. `HTTP.serve` will block on the server listening loop until interrupted or and an irrecoverable error occurs. The `handler` function should be of the form `f(req::HTTP.Request)::HTTP.Response`. Alternatively, passing `stream=true` requires the `handler` to be of the form `f(stream::HTTP.Stream) -> Nothing`. See [`HTTP.Router`](@ref) for details on using it as a request handler. Optional keyword arguments: - `sslconfig=nothing`, Provide an `MbedTLS.SSLConfig` object to handle ssl connections. Pass `sslconfig=MbedTLS.SSLConfig(false)` to disable ssl verification (useful for testing). Construct a custom `SSLConfig` object with `MbedTLS.SSLConfig(certfile, keyfile)`. - `tcpisvalid = tcp->true`, function `f(::TCPSocket)::Bool` to check if accepted connections are valid before processing requests. e.g. to do source IP filtering. - `readtimeout::Int=0`, close the connection if no data is received for this many seconds. Use readtimeout = 0 to disable. - `reuseaddr::Bool=false`, allow multiple servers to listen on the same port. Not supported on some OS platforms. Can check `HTTP.Servers.supportsreuseaddr()`. - `server::Base.IOServer=nothing`, provide an `IOServer` object to listen on; allows manually closing or configuring the server socket. - `verbose::Bool=false`, log connection information to `stdout`. - `access_log::Function`, function for formatting access log messages. The function should accept two arguments, `io::IO` to which the messages should be written, and `http::HTTP.Stream` which can be used to query information from. See also [`@logfmt_str`](@ref). - `on_shutdown::Union{Function, Vector{<:Function}, Nothing}=nothing`, one or more functions to be run if the server is closed (for example by an `InterruptException`). Note, shutdown function(s) will not run if an `IOServer` object is supplied to the `server` keyword argument and closed by `close(server)`. ```julia # start a blocking echo server HTTP.serve("127.0.0.1", 8081) do req return HTTP.Response(200, req.body) end # non-blocking server server = HTTP.serve!(8081) do req return HTTP.Response(200, "response body") end # can gracefully close server manually close(server) ``` """ function serve end """ HTTP.serve!(args...; kw...) -> HTTP.Server Non-blocking version of [`HTTP.serve`](@ref); see that function for details. """ function serve! end serve(f, args...; stream::Bool=false, kw...) = Servers.listen(stream ? f : streamhandler(f), args...; kw...) serve!(f, args...; stream::Bool=false, kw...) = Servers.listen!(stream ? f : streamhandler(f), args...; kw...) # tree-based router handler mutable struct Variable name::String pattern::Union{Nothing, Regex} end const VARREGEX = r"^{([^:{}]+)(?::(.*))?}$" function Variable(pattern) re = Base.match(VARREGEX, pattern) if re === nothing error("problem parsing path variable for route: `$pattern`") end pat = re.captures[2] return Variable(re.captures[1], pat === nothing ? nothing : Regex(pat)) end struct Leaf method::String variables::Vector{Tuple{Int, String}} path::String handler::Any end Base.show(io::IO, x::Leaf) = print(io, "Leaf($(x.method))") export Node mutable struct Node segment::Union{String, Variable} exact::Vector{Node} # sorted alphabetically, all x.segment are String conditional::Vector{Node} # unsorted; will be applied in source-order; all x.segment are Regex wildcard::Union{Node, Nothing} # unconditional variable or wildcard doublestar::Union{Node, Nothing} # /** to match any length of path; must be final segment methods::Vector{Leaf} end Base.show(io::IO, x::Node) = print(io, "Node($(x.segment))") isvariable(x) = startswith(x, "{") && endswith(x, "}") segment(x) = isvariable(x) ? Variable(x) : String(x) Node(x) = Node(x, Node[], Node[], nothing, nothing, Leaf[]) Node() = Node("*") function find(y, itr; by=identity, eq=(==)) for (i, x) in enumerate(itr) eq(by(x), y) && return i end return nothing end function insert!(node::Node, leaf, segments, i) if i > length(segments) # time to insert leaf method match node j = find(leaf.method, node.methods; by=x->x.method, eq=(x, y) -> x == "*" || x == y) if j === nothing push!(node.methods, leaf) else # hmmm, we've seen this route before, warn that we're replacing @warn "replacing existing registered route; $(node.methods[j].method) => \"$(node.methods[j].path)\" route with new path = \"$(leaf.path)\"" node.methods[j] = leaf end return end segment = segments[i] # @show segment, segment isa Variable if segment isa Variable # if we're inserting a variable segment, add variable name to leaf vars array push!(leaf.variables, (i, segment.name)) end # figure out which kind of node this segment is if segment == "*" || (segment isa Variable && segment.pattern === nothing) # wildcard node if node.wildcard === nothing node.wildcard = Node(segment) end return insert!(node.wildcard, leaf, segments, i + 1) elseif segment == "**" # double-star node if node.doublestar === nothing node.doublestar = Node(segment) end if i < length(segments) error("/** double wildcard must be last segment in path") end return insert!(node.doublestar, leaf, segments, i + 1) elseif segment isa Variable # conditional node # check if we've seen this exact conditional segment before j = find(segment.pattern, node.conditional; by=x->x.segment.pattern) if j === nothing # new pattern n = Node(segment) push!(node.conditional, n) else n = node.conditional[j] end return insert!(n, leaf, segments, i + 1) else # exact node @assert segment isa String j = find(segment, node.exact; by=x->x.segment) if j === nothing # new exact match segment n = Node(segment) push!(node.exact, n) sort!(node.exact; by=x->x.segment) return insert!(n, leaf, segments, i + 1) else # existing exact match segment return insert!(node.exact[j], leaf, segments, i + 1) end end end function match(node::Node, method, segments, i) # @show node.segment, i, segments if i > length(segments) if isempty(node.methods) return nothing end j = find(method, node.methods; by=x->x.method, eq=(x, y) -> x == "*" || x == y) if j === nothing # we return missing here so we can return a 405 instead of 404 # i.e. we matched the route, but there wasn't a matching method return missing else # return matched leaf node return node.methods[j] end end segment = segments[i] anymissing = false # first check for exact matches j = find(segment, node.exact; by=x->x.segment) if j !== nothing # found an exact match, recurse m = match(node.exact[j], method, segments, i + 1) anymissing = m === missing m = coalesce(m, nothing) # @show :exact, m if m !== nothing return m end end # check for conditional matches for node in node.conditional # @show node.segment.pattern, segment if Base.match(node.segment.pattern, segment) !== nothing # matched a conditional node, recurse m = match(node, method, segments, i + 1) anymissing = m === missing m = coalesce(m, nothing) if m !== nothing return m end end end if node.wildcard !== nothing m = match(node.wildcard, method, segments, i + 1) anymissing = m === missing m = coalesce(m, nothing) if m !== nothing return m end end if node.doublestar !== nothing m = match(node.doublestar, method, segments, length(segments) + 1) anymissing = m === missing m = coalesce(m, nothing) if m !== nothing return m end end return anymissing ? missing : nothing end """ HTTP.Router(_404, _405, middleware=nothing) Define a router object that maps incoming requests by path to registered routes and associated handlers. Paths can be registered using [`HTTP.register!`](@ref). The router object itself is a "request handler" that can be called like: ``` r = HTTP.Router() resp = r(request) ``` Which will inspect the `request`, find the matching, registered handler from the url, and pass the request on to be handled further. See [`HTTP.register!`](@ref) for additional information on registering handlers based on routes. If a request doesn't have a matching, registered handler, the `_404` handler is called which, by default, returns a `HTTP.Response(404)`. If a route matches the path, but not the method/verb (e.g. there's a registerd route for "GET /api", but the request is "POST /api"), then the `_405` handler is called, which by default returns `HTTP.Response(405)` (method not allowed). A `middleware` ([`Middleware`](@ref)) can optionally be provided as well, which will be called after the router has matched the request to a route, but before the route's handler is called. This provides a "hook" for matched routes that can be helpful for metric tracking, logging, etc. Note that the middleware is only called if the route is matched; for the 404 and 405 cases, users should wrap those handlers in the `middleware` manually. """ struct Router{T, S, F} _404::T _405::S routes::Node middleware::F end default404(::Request) = Response(404) default405(::Request) = Response(405) default404(s::Stream) = setstatus(s, 404) default405(s::Stream) = setstatus(s, 405) Router(_404=default404, _405=default405, middleware=nothing) = Router(_404, _405, Node(), middleware) """ HTTP.register!(r::Router, method, path, handler) HTTP.register!(r::Router, path, handler) Register a handler function that should be called when an incoming request matches `path` and the optionally provided `method` (if not provided, any method is allowed). Can be used to dynamically register routes. When a registered route is matched, the original route string is stored in the `request.context[:route]` variable. The following path types are allowed for matching: * `/api/widgets`: exact match of static strings * `/api/*/owner`: single `*` to wildcard match anything for a single segment * `/api/widget/{id}`: Define a path variable `id` that matches any valued provided for this segment; path variables are available in the request context like `HTTP.getparams(req)["id"]` * `/api/widget/{id:[0-9]+}`: Define a path variable `id` that does a regex match for integers for this segment * `/api/**`: double wildcard matches any number of trailing segments in the request path; the double wildcard must be the last segment in the path """ function register! end function register!(r::Router, method, path, handler) segments = map(segment, split(path, '/'; keepempty=false)) if r.middleware !== nothing handler = r.middleware(handler) end insert!(r.routes, Leaf(method, Tuple{Int, String}[], path, handler), segments, 1) return end register!(r::Router, path, handler) = register!(r, "*", path, handler) const Params = Dict{String, String} function gethandler(r::Router, req::Request) url = URI(req.target) segments = split(url.path, '/'; keepempty=false) leaf = match(r.routes, req.method, segments, 1) params = Params() if leaf isa Leaf # @show leaf.variables, segments if !isempty(leaf.variables) # we have variables to fill in for (i, v) in leaf.variables params[v] = segments[i] end end return leaf.handler, leaf.path, params end return leaf, "", params end function (r::Router)(stream::Stream{<:Request}) req = stream.message handler, route, params = gethandler(r, req) if handler === nothing # didn't match a registered route return r._404(stream) elseif handler === missing # matched the path, but method not supported return r._405(stream) else req.context[:route] = route if !isempty(params) req.context[:params] = params end return handler(stream) end end function (r::Router)(req::Request) handler, route, params = gethandler(r, req) if handler === nothing # didn't match a registered route return r._404(req) elseif handler === missing # matched the path, but method not supported return r._405(req) else req.context[:route] = route if !isempty(params) req.context[:params] = params end return handler(req) end end """ HTTP.getroute(req) -> String Retrieve the original route registration string for a request after its url has been matched against a router. Helpful for metric logging to ignore matched variables in a path and only see the registered routes. """ getroute(req) = get(req.context, :route, nothing) """ HTTP.getparams(req) -> Dict{String, String} Retrieve any matched path parameters from the request context. If a path was registered with a router via `HTTP.register!` like "/api/widget/{id}", then the path parameters are available in the request context and can be retrieved like `id = HTTP.getparams(req)["id"]`. """ getparams(req) = get(req.context, :params, nothing) """ HTTP.getparam(req, name, default=nothing) -> String Retrieve a matched path parameter with name `name` from request context. If a path was registered with a router via `HTTP.register!` like "/api/widget/{id}", then the path parameter can be retrieved like `id = HTTP.getparam(req, "id"). """ function getparam(req, name, default=nothing) params = getparams(req) params === nothing && return default return get(params, name, default) end """ HTTP.Handlers.cookie_middleware(handler) -> handler Middleware that parses and stores any cookies in the incoming request in the request context. Cookies can then be retrieved by calling [`HTTP.getcookies(req)`](@ref) in subsequent middlewares/handlers. """ function cookie_middleware(handler) function (req) if !haskey(req.context, :cookies) req.context[:cookies] = Cookies.cookies(req) end return handler(req) end end """ HTTP.getcookies(req) -> Vector{Cookie} Retrieve any parsed cookies from a request context. Cookies are expected to be stored in the `req.context[:cookies]` of the request context as implemented in the [`HTTP.Handlers.cookie_middleware`](@ref) middleware. """ getcookies(req) = get(() -> Cookie[], req.context, :cookies) end # module 4/opt/julia/packages/HTTP/sJD5V/src/parsemultipart.jl%module MultiPartParsing import ..access_threaded using ..Messages, ..Forms, ..Parsers export parse_multipart_form const CR_BYTE = 0x0d # \r const LF_BYTE = 0x0a # \n const DASH_BYTE = 0x2d # - const HTAB_BYTE = 0x09 # \t const SPACE_BYTE = 0x20 const SEMICOLON_BYTE = UInt8(';') const CRLFCRLF = (CR_BYTE, LF_BYTE, CR_BYTE, LF_BYTE) "compare byte buffer `a` from index `i` to index `j` with `b` and check if they are byte-equal" function byte_buffers_eq(a, i, j, b) l = 1 @inbounds for k = i:j a[k] == b[l] || return false l += 1 end return true end """ find_multipart_boundary(bytes, boundaryDelimiter; start::Int=1) Find the first and last index of the next boundary delimiting a part, and if the discovered boundary is the terminating boundary. """ function find_multipart_boundary(bytes::AbstractVector{UInt8}, boundaryDelimiter::AbstractVector{UInt8}; start::Int=1) # The boundary delimiter line is prepended with two '-' characters # The boundary delimiter line starts on a new line, so must be preceded by a \r\n. # The boundary delimiter line ends with \r\n, and can have "optional linear whitespace" between # the end of the boundary delimiter, and the \r\n. # The last boundary delimiter line has an additional '--' at the end of the boundary delimiter # [RFC2046 5.1.1](https://tools.ietf.org/html/rfc2046#section-5.1.1) i = start end_index = i + length(boundaryDelimiter) + 1 while end_index <= length(bytes) if bytes[i] == DASH_BYTE && bytes[i + 1] == DASH_BYTE && byte_buffers_eq(bytes, i + 2, end_index, boundaryDelimiter) # boundary delimiter line start on a new line ... if i > 1 (i == 2 || bytes[i-2] != CR_BYTE || bytes[i-1] != LF_BYTE) && error("boundary delimiter found, but it was not the start of a line") # the CRLF preceding the boundary delimiter is "conceptually attached # to the boundary", so account for this with the index i -= 2 end # need to check if there are enough characters for the CRLF or for two dashes end_index < length(bytes)-1 || error("boundary delimiter found, but did not end with new line") is_terminating_delimiter = bytes[end_index+1] == DASH_BYTE && bytes[end_index+2] == DASH_BYTE is_terminating_delimiter && (end_index += 2) # ... there can be arbitrary SP and HTAB space between the boundary delimiter ... while end_index < length(bytes) && (bytes[end_index+1] in (HTAB_BYTE, SPACE_BYTE)) end_index += 1 end # ... and ends with a new line newlineEnd = end_index < length(bytes)-1 && bytes[end_index+1] == CR_BYTE && bytes[end_index+2] == LF_BYTE if !newlineEnd error("boundary delimiter found, but did not end with new line") end end_index += 2 return (is_terminating_delimiter, i, end_index) end i += 1 end_index += 1 end error("boundary delimiter not found") end """ find_multipart_boundaries(bytes, boundary; start=1) Find the start and end indexes of all the parts of the multipart object. Ultimately this method is looking for the data between the boundary delimiters in the byte array. A vector containing all the start/end pairs is returned. """ function find_multipart_boundaries(bytes::AbstractVector{UInt8}, boundary::AbstractVector{UInt8}; start=1) idxs = Tuple{Int, Int}[] while true (is_terminating_delimiter, i, end_index) = find_multipart_boundary(bytes, boundary; start = start) push!(idxs, (i, end_index)) is_terminating_delimiter && break start = end_index + 1 end return idxs end """ find_header_boundary(bytes) Find the end of the multipart header in the byte array. Returns a Tuple with the start index(1) and the end index. Headers are separated from the body by CRLFCRLF. [RFC2046 5.1](https://tools.ietf.org/html/rfc2046#section-5.1) [RFC822 3.1](https://tools.ietf.org/html/rfc822#section-3.1) """ function find_header_boundary(bytes::AbstractVector{UInt8}) length(CRLFCRLF) > length(bytes) && return nothing l = length(bytes) - length(CRLFCRLF) + 1 i = 1 end_index = length(CRLFCRLF) while (i <= l) byte_buffers_eq(bytes, i, end_index, CRLFCRLF) && return (1, end_index) i += 1 end_index += 1 end error("no delimiter found separating header from multipart body") end const content_disposition_regex = Parsers.RegexAndMatchData[] function content_disposition_regex_f() r = Parsers.RegexAndMatchData(r"^[Cc]ontent-[Dd]isposition:[ \t]*form-data;[ \t]*(.*)\r\n"mx) Parsers.init!(r) end const content_disposition_flag_regex = Parsers.RegexAndMatchData[] function content_disposition_flag_regex_f() r = Parsers.RegexAndMatchData(r"""^ [ \t]*([!#$%&'*+\-.^_`|~[:alnum:]]+);? """x) Parsers.init!(r) end const content_disposition_pair_regex = Parsers.RegexAndMatchData[] function content_disposition_pair_regex_f() r = Parsers.RegexAndMatchData(r"""^ [ \t]*([!#$%&'*+\-.^_`|~[:alnum:]]+)[ \t]*=[ \t]*"(.*?)";? """x) Parsers.init!(r) end const content_type_regex = Parsers.RegexAndMatchData[] function content_type_regex_f() r = Parsers.RegexAndMatchData(r"(?i)Content-Type: (\S*[^;\s])") Parsers.init!(r) end """ parse_multipart_chunk(chunk) Parse a single multi-part chunk into a Multipart object. This will decode the header and extract the contents from the byte array. """ function parse_multipart_chunk(chunk) startIndex, end_index = find_header_boundary(chunk) header = SubString(unsafe_string(pointer(chunk, startIndex), end_index - startIndex + 1)) content = view(chunk, end_index+1:lastindex(chunk)) # find content disposition re = access_threaded(content_disposition_regex_f, content_disposition_regex) if !Parsers.exec(re, header) @warn "Content disposition is not specified dropping the chunk." String(chunk) return nothing # Specifying content disposition is mandatory end content_disposition = Parsers.group(1, re, header) re_flag = access_threaded(content_disposition_flag_regex_f, content_disposition_flag_regex) re_pair = access_threaded(content_disposition_pair_regex_f, content_disposition_pair_regex) name = nothing filename = nothing while !isempty(content_disposition) if Parsers.exec(re_pair, content_disposition) key = Parsers.group(1, re_pair, content_disposition) value = Parsers.group(2, re_pair, content_disposition) if key == "name" name = value elseif key == "filename" filename = value else # do stuff with other content disposition key-value pairs end content_disposition = Parsers.nextbytes(re_pair, content_disposition) elseif Parsers.exec(re_flag, content_disposition) # do stuff with content disposition flags content_disposition = Parsers.nextbytes(re_flag, content_disposition) else break end end name === nothing && return re_ct = access_threaded(content_type_regex_f, content_type_regex) contenttype = Parsers.exec(re_ct, header) ? Parsers.group(1, re_ct, header) : "text/plain" return Multipart(filename, IOBuffer(content), contenttype, "", name) end """ parse_multipart_body(body, boundary)::Vector{Multipart} Parse the multipart body received from the client breaking it into the various chunks which are returned as an array of Multipart objects. """ function parse_multipart_body(body::AbstractVector{UInt8}, boundary::AbstractString)::Vector{Multipart} multiparts = Multipart[] idxs = find_multipart_boundaries(body, codeunits(boundary)) length(idxs) > 1 || (return multiparts) for i in 1:length(idxs)-1 chunk = view(body, idxs[i][2]+1:idxs[i+1][1]-1) push!(multiparts, parse_multipart_chunk(chunk)) end return multiparts end """ parse_multipart_form(req::Request)::Vector{Multipart} Parse the full mutipart form submission from the client returning and array of Multipart objects containing all the data. The order of the multipart form data in the request should be preserved. [RFC7578 5.2](https://tools.ietf.org/html/rfc7578#section-5.2). The boundary delimiter MUST NOT appear inside any of the encapsulated parts. Note that the boundary delimiter does not need to have '-' characters, but a line using the boundary delimiter will start with '--' and end in \r\n. [RFC2046 5.1](https://tools.ietf.org/html/rfc2046#section-5.1.1) """ function parse_multipart_form(msg::Message)::Union{Vector{Multipart}, Nothing} # parse boundary from Content-Type m = match(r"multipart/form-data; boundary=(.*)$", msg["Content-Type"]) m === nothing && return nothing boundary_delimiter = m[1] # [RFC2046 5.1.1](https://tools.ietf.org/html/rfc2046#section-5.1.1) length(boundary_delimiter) > 70 && error("boundary delimiter must not be greater than 70 characters") return parse_multipart_body(payload(msg), boundary_delimiter) end function __init__() nt = isdefined(Base.Threads, :maxthreadid) ? Threads.maxthreadid() : Threads.nthreads() resize!(empty!(content_disposition_regex), nt) resize!(empty!(content_disposition_flag_regex), nt) resize!(empty!(content_disposition_pair_regex), nt) resize!(empty!(content_type_regex), nt) return end end # module MultiPartParsing 0/opt/julia/packages/HTTP/sJD5V/src/WebSockets.jl lmodule WebSockets using Base64, LoggingExtras, UUIDs, Sockets, Random using MbedTLS: digest, MD_SHA1, SSLContext using ..IOExtras, ..Streams, ..Connections, ..Messages, ..Conditions, ..Servers using ..Exceptions: current_exceptions_to_string import ..open import ..HTTP # for doc references export WebSocket, send, receive, ping, pong # 1st 2 bytes of a frame primitive type FrameFlags 16 end uint16(x::FrameFlags) = Base.bitcast(UInt16, x) FrameFlags(x::UInt16) = Base.bitcast(FrameFlags, x) const WS_FINAL = 0b1000000000000000 const WS_RSV1 = 0b0100000000000000 const WS_RSV2 = 0b0010000000000000 const WS_RSV3 = 0b0001000000000000 const WS_OPCODE = 0b0000111100000000 const WS_MASK = 0b0000000010000000 const WS_LEN = 0b0000000001111111 @enum OpCode::UInt8 CONTINUATION=0x00 TEXT=0x01 BINARY=0x02 CLOSE=0x08 PING=0x09 PONG=0x0A iscontrol(opcode::OpCode) = opcode > BINARY Base.propertynames(x::FrameFlags) = (:final, :rsv1, :rsv2, :rsv3, :opcode, :mask, :len) function Base.getproperty(x::FrameFlags, nm::Symbol) ux = uint16(x) if nm == :final return ux & WS_FINAL > 0 elseif nm == :rsv1 return ux & WS_RSV1 > 0 elseif nm == :rsv2 return ux & WS_RSV2 > 0 elseif nm == :rsv3 return ux & WS_RSV3 > 0 elseif nm == :opcode return OpCode(((ux & WS_OPCODE) >> 8) % UInt8) elseif nm == :masked return ux & WS_MASK > 0 elseif nm == :len return ux & WS_LEN end end FrameFlags(final::Bool, opcode::OpCode, masked::Bool, len::Integer; rsv1::Bool=false, rsv2::Bool=false, rsv3::Bool=false) = FrameFlags( (final ? WS_FINAL : UInt16(0)) | (rsv1 ? WS_RSV1 : UInt16(0)) | (rsv2 ? WS_RSV2 : UInt16(0)) | (rsv3 ? WS_RSV3 : UInt16(0)) | (UInt16(opcode) << 8) | (masked ? WS_MASK : UInt16(0)) | (len % UInt16) ) Base.show(io::IO, x::FrameFlags) = print(io, "FrameFlags(", "final=", x.final, ", ", "opcode=", x.opcode, ", ", "masked=", x.masked, ", ", "len=", x.len, ")") primitive type Mask 32 end Base.UInt32(x::Mask) = Base.bitcast(UInt32, x) Mask(x::UInt32) = Base.bitcast(Mask, x) Base.getindex(x::Mask, i::Int) = (UInt32(x) >> (8 * ((i - 1) % 4))) % UInt8 mask() = Mask(rand(Random.RandomDevice(), UInt32)) const EMPTY_MASK = Mask(UInt32(0)) # representation of a single websocket frame struct Frame flags::FrameFlags extendedlen::Union{Nothing, UInt16, UInt64} mask::Mask # when sending, Vector{UInt8} if client, any AbstractVector{UInt8} if server # when receiving: # CONTINUATION: String or Vector{UInt8} based on first fragment frame opcode TEXT/BINARY # TEXT: String # BINARY/PING/PONG: Vector{UInt8} # CLOSE: CloseFrameBody payload::Any end # given a payload total length, split into 7-bit length + 16-bit or 64-bit extended length wslength(l) = l < 0x7E ? (UInt8(l), nothing) : l <= 0xFFFF ? (0x7E, UInt16(l)) : (0x7F, UInt64(l)) # give a mutable byte payload + mask, perform client websocket masking function mask!(bytes::Vector{UInt8}, mask) for i in 1:length(bytes) @inbounds bytes[i] = bytes[i] ⊻ mask[i] end return end # send method Frame constructor function Frame(final::Bool, opcode::OpCode, client::Bool, payload::AbstractVector{UInt8}; rsv1::Bool=false, rsv2::Bool=false, rsv3::Bool=false) len, extlen = wslength(length(payload)) if client msk = mask() mask!(payload, msk) else msk = EMPTY_MASK end return Frame(FrameFlags(final, opcode, client, len; rsv1, rsv2, rsv3), extlen, msk, payload) end Base.show(io::IO, x::Frame) = print(io, "Frame(", "flags=", x.flags, ", ", "extendedlen=", x.extendedlen, ", ", "mask=", x.mask, ", ", "payload=", x.payload, ")") # reading a single frame # If _The WebSocket Connection is Closed_ and no Close control frame was received by the # endpoint (such as could occur if the underlying transport connection # is lost), _The WebSocket Connection Close Code_ is considered to be 1006. @noinline iocheck(io) = isopen(io) || throw(WebSocketError(CloseFrameBody(1006, "WebSocket connection is closed"))) """ WebSockets.readframe(ws) -> WebSockets.Frame WebSockets.readframe(io, Frame, buffer, first_fragment_opcode) -> WebSockets.Frame Read a single websocket frame from a `WebSocket` or `IO` stream. Frame may be a control frame with `PING`, `PONG`, or `CLOSE` opcode. Frame may also be part of fragmented message, with opcdoe `CONTINUATION`; `first_fragment_opcode` should be passed from the 1st frame of a fragmented message to ensure each subsequent frame payload is converted correctly (String or Vector{UInt8}). """ function readframe(io::IO, ::Type{Frame}, buffer::Vector{UInt8}=UInt8[], first_fragment_opcode::OpCode=CONTINUATION) iocheck(io) flags = FrameFlags(ntoh(read(io, UInt16))) if flags.len == 0x7E extlen = ntoh(read(io, UInt16)) len = UInt64(extlen) elseif flags.len == 0x7F extlen = ntoh(read(io, UInt64)) len = extlen else extlen = nothing len = UInt64(flags.len) end mask = flags.masked ? Mask(read(io, UInt32)) : EMPTY_MASK # even if len is 0, we need to resize! so previously filled buffers aren't erroneously reused resize!(buffer, len) if len > 0 # NOTE: we could support a pure streaming case by allowing the caller to pass # an IO instead of buffer and writing directly from io -> out_io. # The tricky case would be server-side streaming, where we need to unmask # the incoming client payload; we could just buffer the payload + unmask # and then write out to the out_io. read!(io, buffer) end if flags.masked mask!(buffer, mask) end if flags.opcode == CONTINUATION && first_fragment_opcode == CONTINUATION throw(WebSocketError(CloseFrameBody(1002, "Continuation frame cannot be the first frame in a message"))) elseif first_fragment_opcode != CONTINUATION && flags.opcode in (TEXT, BINARY) throw(WebSocketError(CloseFrameBody(1002, "Received unfragmented frame while still processing fragmented frame"))) end op = flags.opcode == CONTINUATION ? first_fragment_opcode : flags.opcode if op == TEXT # TODO: possible avoid the double copy from read!(io, buffer) + unsafe_string? payload = unsafe_string(pointer(buffer), len) elseif op == CLOSE if len == 1 throw(WebSocketError(CloseFrameBody(1002, "Close frame cannot have body of length 1"))) end control_len_check(len) if len >= 2 st = Int(UInt16(buffer[1]) << 8 | buffer[2]) validclosecheck(st) status = st else status = 1005 end payload = CloseFrameBody(status, len > 2 ? unsafe_string(pointer(buffer) + 2, len - 2) : "") utf8check(payload.message) else # BINARY payload = copy(buffer) end return Frame(flags, extlen, mask, payload) end # writing a single frame function writeframe(io::IO, x::Frame) n = write(io.io, hton(uint16(x.flags))) if x.extendedlen !== nothing n += write(io.io, hton(x.extendedlen)) end if x.mask != EMPTY_MASK n += write(io.io, UInt32(x.mask)) end pl = x.payload # manually unroll a few known type cases to help the compiler if pl isa Vector{UInt8} n += write(io.io, pl) elseif pl isa Base.CodeUnits{UInt8, String} n += write(io.io, pl) else n += write(io.io, pl) end return n end "Status codes according to RFC 6455 7.4.1" const STATUS_CODE_DESCRIPTION = Dict{Int, String}( 1000=>"Normal", 1001=>"Going Away", 1002=>"Protocol Error", 1003=>"Unsupported Data", 1004=>"Reserved", 1005=>"No Status Recvd- reserved", 1006=>"Abnormal Closure- reserved", 1007=>"Invalid frame payload data", 1008=>"Policy Violation", 1009=>"Message too big", 1010=>"Missing Extension", 1011=>"Internal Error", 1012=>"Service Restart", 1013=>"Try Again Later", 1014=>"Bad Gateway", 1015=>"TLS Handshake") @noinline validclosecheck(x) = (1000 <= x < 5000 && !(x in (1004, 1005, 1006, 1016, 1100, 2000, 2999))) || throw(WebSocketError(CloseFrameBody(1002, "Invalid close status code"))) """ WebSockets.CloseFrameBody(status, message) Represents the payload of a CLOSE control websocket frame. For error close `status`, it can be wrapped in a `WebSocketError` and thrown. """ struct CloseFrameBody status::Int message::String end struct WebSocketError <: Exception message::Union{String, CloseFrameBody} end """ WebSockets.isok(x::WebSocketError) -> Bool Returns true if the `WebSocketError` has a non-error status code. When calling `receive(websocket)`, if a CLOSE frame is received, the CLOSE frame body is parsed and thrown inside the `WebSocketError`, but if the CLOSE frame has a non-error status code, it's safe to ignore the error and return from the `WebSockets.open` or `WebSockets.listen` calls without throwing. """ isok(x) = x isa WebSocketError && x.message isa CloseFrameBody && (x.message.status == 1000 || x.message.status == 1001 || x.message.status == 1005) """ WebSocket(io::HTTP.Connection, req, resp; client=true) Representation of a websocket connection. Use `WebSockets.open` to open a websocket connection, passing a handler function `f(ws)` to send and receive messages. Use `WebSockets.listen` to listen for incoming websocket connections, passing a handler function `f(ws)` to send and receive messages. Call `send(ws, msg)` to send a message; if `msg` is an `AbstractString`, a TEXT websocket message will be sent; if `msg` is an `AbstractVector{UInt8}`, a BINARY websocket message will be sent. Otherwise, `msg` should be an iterable of either `AbstractString` or `AbstractVector{UInt8}`, and a fragmented message will be sent, one frame for each iterated element. Control frames can be sent by calling `ping(ws[, data])`, `pong(ws[, data])`, or `close(ws[, body::WebSockets.CloseFrameBody])`. Calling `close` will initiate the close sequence and close the underlying connection. To receive messages, call `receive(ws)`, which will block until a non-control, full message is received. PING messages will automatically be responded to when received. CLOSE messages will also be acknowledged and then a `WebSocketError` will be thrown with the `WebSockets.CloseFrameBody` payload, which may include a non-error CLOSE frame status code. `WebSockets.isok(err)` can be called to check if the CLOSE was normal or unexpected. Fragmented messages will be received until the final frame is received and the full concatenated payload can be returned. `receive(ws)` returns a `Vector{UInt8}` for BINARY messages, and a `String` for TEXT messages. For convenience, `WebSocket`s support the iteration protocol, where each iteration will `receive` a non-control message, with iteration terminating when the connection is closed. E.g.: ```julia WebSockets.open(url) do ws for msg in ws # do cool stuff with msg end end ``` """ mutable struct WebSocket id::UUID io::Connection request::Request response::Response maxframesize::Int maxfragmentation::Int client::Bool readbuffer::Vector{UInt8} writebuffer::Vector{UInt8} readclosed::Bool writeclosed::Bool end const DEFAULT_MAX_FRAG = 1024 IOExtras.tcpsocket(ws::WebSocket) = tcpsocket(ws.io) WebSocket(io::Connection, req=Request(), resp=Response(); client::Bool=true, maxframesize::Integer=typemax(Int), maxfragmentation::Integer=DEFAULT_MAX_FRAG) = WebSocket(uuid4(), io, req, resp, maxframesize, maxfragmentation, client, UInt8[], UInt8[], false, false) """ WebSockets.isclosed(ws) -> Bool Check whether a `WebSocket` has sent and received CLOSE frames. """ isclosed(ws::WebSocket) = ws.readclosed && ws.writeclosed # Handshake "Check whether a HTTP.Request or HTTP.Response is a websocket upgrade request/response" function isupgrade(r::Message) ((r isa Request && r.method == "GET") || (r isa Response && r.status == 101)) && (hasheader(r, "Connection", "upgrade") || hasheader(r, "Connection", "keep-alive, upgrade")) && hasheader(r, "Upgrade", "websocket") end # Renamed in HTTP@1 @deprecate is_upgrade isupgrade @noinline handshakeerror() = throw(WebSocketError(CloseFrameBody(1002, "Websocket handshake failed"))) function hashedkey(key) hashkey = "$(strip(key))258EAFA5-E914-47DA-95CA-C5AB0DC85B11" return base64encode(digest(MD_SHA1, hashkey)) end """ WebSockets.open(handler, url; verbose=false, kw...) Initiate a websocket connection to `url` (which should have schema like `ws://` or `wss://`), and call `handler(ws)` with the websocket connection. Passing `verbose=true` or `verbose=2` will enable debug logging for the life of the websocket connection. `handler` should be a function of the form `f(ws) -> nothing`, where `ws` is a [`WebSocket`](@ref). Supported keyword arguments are the same as supported by [`HTTP.request`](@ref). Typical websocket usage is: ```julia WebSockets.open(url) do ws # iterate incoming websocket messages for msg in ws # send message back to server or do other logic here send(ws, msg) end # iteration ends when the websocket connection is closed by server or error end ``` """ function open(f::Function, url; suppress_close_error::Bool=false, verbose=false, headers=[], maxframesize::Integer=typemax(Int), maxfragmentation::Integer=DEFAULT_MAX_FRAG, kw...) key = base64encode(rand(Random.RandomDevice(), UInt8, 16)) headers = [ "Upgrade" => "websocket", "Connection" => "Upgrade", "Sec-WebSocket-Key" => key, "Sec-WebSocket-Version" => "13", headers... ] # HTTP.open open("GET", url, headers; verbose=verbose, kw...) do http startread(http) isupgrade(http.message) || handshakeerror() if header(http, "Sec-WebSocket-Accept") != hashedkey(key) throw(WebSocketError("Invalid Sec-WebSocket-Accept\n" * "$(http.message)")) end # later stream logic checks to see if the HTTP message is "complete" # by seeing if ntoread is 0, which is typemax(Int) for websockets by default # so set it to 0 so it's correctly viewed as "complete" once we're done # doing websocket things http.ntoread = 0 io = http.stream ws = WebSocket(io, http.message.request, http.message; maxframesize, maxfragmentation) @debugv 2 "$(ws.id): WebSocket opened" try f(ws) catch e if !isok(e) suppress_close_error || @error "$(ws.id): error" (e, catch_backtrace()) end if !isclosed(ws) if e isa WebSocketError && e.message isa CloseFrameBody close(ws, e.message) else close(ws, CloseFrameBody(1008, "Unexpected client websocket error")) end end if !isok(e) rethrow() end finally if !isclosed(ws) close(ws, CloseFrameBody(1000, "")) end end end end """ WebSockets.listen(handler, host, port; verbose=false, kw...) WebSockets.listen!(handler, host, port; verbose=false, kw...) -> HTTP.Server Listen for websocket connections on `host` and `port`, and call `handler(ws)`, which should be a function taking a single `WebSocket` argument. Keyword arguments `kw...` are the same as supported by [`HTTP.listen`](@ref). Typical usage is like: ```julia WebSockets.listen(host, port) do ws # iterate incoming websocket messages for msg in ws # send message back to client or do other logic here send(ws, msg) end # iteration ends when the websocket connection is closed by client or error end ``` """ function listen end listen(f, args...; kw...) = Servers.listen(http -> upgrade(f, http; kw...), args...; kw...) listen!(f, args...; kw...) = Servers.listen!(http -> upgrade(f, http; kw...), args...; kw...) function upgrade(f::Function, http::Streams.Stream; suppress_close_error::Bool=false, maxframesize::Integer=typemax(Int), maxfragmentation::Integer=DEFAULT_MAX_FRAG, nagle=false, quickack=true, kw...) @debugv 2 "Server websocket upgrade requested" isupgrade(http.message) || handshakeerror() if !hasheader(http, "Sec-WebSocket-Version", "13") throw(WebSocketError("Expected \"Sec-WebSocket-Version: 13\"!\n" * "$(http.message)")) end if !hasheader(http, "Sec-WebSocket-Key") throw(WebSocketError("Expected \"Sec-WebSocket-Key header\"!\n" * "$(http.message)")) end setstatus(http, 101) setheader(http, "Upgrade" => "websocket") setheader(http, "Connection" => "Upgrade") key = header(http, "Sec-WebSocket-Key") setheader(http, "Sec-WebSocket-Accept" => hashedkey(key)) startwrite(http) io = http.stream req = http.message # tune websocket tcp connection for performance : https://github.com/JuliaWeb/HTTP.jl/issues/1140 @static if VERSION >= v"1.3" sock = tcpsocket(io) # I don't understand why uninitializd sockets can get here, but they can if sock.status ∉ (Base.StatusInit, Base.StatusUninit) && isopen(sock) Sockets.nagle(sock, nagle) Sockets.quickack(sock, quickack) end end ws = WebSocket(io, req, req.response; client=false, maxframesize, maxfragmentation) @debugv 2 "$(ws.id): WebSocket upgraded; connection established" try f(ws) catch e if !isok(e) suppress_close_error || @error begin msg = current_exceptions_to_string() "$(ws.id): Unexpected websocket server error. $msg" end end if !isclosed(ws) if e isa WebSocketError && e.message isa CloseFrameBody close(ws, e.message) else close(ws, CloseFrameBody(1011, "Unexpected server websocket error")) end end if !isok(e) rethrow() end finally if !isclosed(ws) close(ws, CloseFrameBody(1000, "")) end end end # Sending messages isbinary(x) = x isa AbstractVector{UInt8} istext(x) = x isa AbstractString opcode(x) = isbinary(x) ? BINARY : TEXT function payload(ws, x) if ws.client # if we're client, we need to mask the payload, so use our writebuffer for masking pload = isbinary(x) ? x : codeunits(string(x)) len = length(pload) resize!(ws.writebuffer, len) copyto!(ws.writebuffer, pload) return ws.writebuffer else # if we're server, we just need to make sure payload is AbstractVector{UInt8} return isbinary(x) ? x : codeunits(string(x)) end end """ send(ws::WebSocket, msg) Send a message on a websocket connection. If `msg` is an `AbstractString`, a TEXT websocket message will be sent; if `msg` is an `AbstractVector{UInt8}`, a BINARY websocket message will be sent. Otherwise, `msg` should be an iterable of either `AbstractString` or `AbstractVector{UInt8}`, and a fragmented message will be sent, one frame for each iterated element. Control frames can be sent by calling `ping(ws[, data])`, `pong(ws[, data])`, or `close(ws[, body::WebSockets.CloseFrameBody])`. Calling `close` will initiate the close sequence and close the underlying connection. """ function Sockets.send(ws::WebSocket, x) @debugv 2 "$(ws.id): Writing non-control message" @require !ws.writeclosed if !isbinary(x) && !istext(x) # if x is not single binary or text, then assume it's an iterable of binary or text # and we'll send fragmented message first = true n = 0 state = iterate(x) if state === nothing # x was not binary or text, but is an empty iterable, send single empty frame x = "" @goto write_single_frame end @debugv 2 "$(ws.id): Writing fragmented message" item, st = state # we prefetch next state so we know if we're on the last item or not # so we can appropriately set the FIN bit for the last fragmented frame nextstate = iterate(x, st) while true n += writeframe(ws.io, Frame(nextstate === nothing, first ? opcode(item) : CONTINUATION, ws.client, payload(ws, item))) first = false nextstate === nothing && break item, st = nextstate nextstate = iterate(x, st) end else # single binary or text frame for message @label write_single_frame return writeframe(ws.io, Frame(true, opcode(x), ws.client, payload(ws, x))) end end # control frames """ ping(ws, data=[]) Send a PING control frame on a websocket connection. `data` is an optional body to send with the message. PONG messages are automatically responded to when a PING message is received by a websocket connection. """ function ping(ws::WebSocket, data=UInt8[]) @require !ws.writeclosed @debugv 2 "$(ws.id): sending ping" return writeframe(ws.io, Frame(true, PING, ws.client, payload(ws, data))) end """ pong(ws, data=[]) Send a PONG control frame on a websocket connection. `data` is an optional body to send with the message. Note that PING messages are automatically responded to internally by the websocket connection with a corresponding PONG message, but in certain cases, a unidirectional PONG message can be used as a one-way heartbeat. """ function pong(ws::WebSocket, data=UInt8[]) @require !ws.writeclosed @debugv 2 "$(ws.id): sending pong" return writeframe(ws.io, Frame(true, PONG, ws.client, payload(ws, data))) end """ close(ws, body::WebSockets.CloseFrameBody=nothing) Initiate a close sequence on a websocket connection. `body` is an optional `WebSockets.CloseFrameBody` with a status code and optional reason message. If a CLOSE frame has already been received, then a responding CLOSE frame is sent and the connection is closed. If a CLOSE frame hasn't already been received, the CLOSE frame is sent and `receive` is attempted to receive the responding CLOSE frame. """ function Base.close(ws::WebSocket, body::CloseFrameBody=CloseFrameBody(1000, "")) isclosed(ws) && return @debugv 2 "$(ws.id): Closing websocket" ws.writeclosed = true data = Vector{UInt8}(body.message) prepend!(data, reinterpret(UInt8, [hton(UInt16(body.status))])) try writeframe(ws.io, Frame(true, CLOSE, ws.client, data)) catch # ignore thrown errors here because we're closing anyway end # if we're initiating the close, wait until we receive the # responding close frame or timeout if !ws.readclosed Timer(5) do t ws.readclosed = true !ws.client && isopen(ws.io) && close(ws.io) end end while !ws.readclosed try receive(ws) catch # ignore thrown errors here because we're closing anyway # but set readclosed so we don't keep trying to read ws.readclosed = true end end # we either recieved the responding CLOSE frame and readclosed was set # or there was an error/timeout reading it; in any case, readclosed should be closed now @assert ws.readclosed # if we're the server, it's our job to close the underlying socket !ws.client && isopen(ws.io) && close(ws.io) return end # Receiving messages # returns whether additional frames should be read # true if fragmented message or a ping/pong frame was handled @noinline control_len_check(len) = len > 125 && throw(WebSocketError(CloseFrameBody(1002, "Invalid length for control frame"))) @noinline utf8check(x) = isvalid(x) || throw(WebSocketError(CloseFrameBody(1007, "Invalid UTF-8"))) function checkreadframe!(ws::WebSocket, frame::Frame) if frame.flags.rsv1 || frame.flags.rsv2 || frame.flags.rsv3 throw(WebSocketError(CloseFrameBody(1002, "Reserved bits set in control frame"))) end opcode = frame.flags.opcode if iscontrol(opcode) && !frame.flags.final throw(WebSocketError(CloseFrameBody(1002, "Fragmented control frame"))) end if opcode == CLOSE ws.readclosed = true # reply with Close control frame if we didn't initiate close if !ws.writeclosed close(ws) end throw(WebSocketError(frame.payload)) elseif opcode == PING control_len_check(frame.flags.len) pong(ws, frame.payload) return false elseif opcode == PONG control_len_check(frame.flags.len) return false elseif frame.flags.final && frame.flags.opcode == TEXT && frame.payload isa String utf8check(frame.payload) end return frame.flags.final end _append(x::AbstractVector{UInt8}, y::AbstractVector{UInt8}) = append!(x, y) _append(x::String, y::String) = string(x, y) # low-level for reading a single frame readframe(ws::WebSocket) = readframe(ws.io, Frame, ws.readbuffer) """ receive(ws::WebSocket) -> Union{String, Vector{UInt8}} Receive a message from a websocket connection. Returns a `String` if the message was TEXT, or a `Vector{UInt8}` if the message was BINARY. If control frames (ping or pong) are received, they are handled automatically and a non-control message is waited for. If a CLOSE message is received, it is responded to and a `WebSocketError` is thrown with the `WebSockets.CloseFrameBody` as the error value. This error can be checked with `WebSockets.isok(err)` to see if the closing was "normal" or if an actual error occurred. For fragmented messages, the incoming frames will continue to be read until the final fragment is received. The bodies of each fragment are concatenated into the final message returned by `receive`. Note that `WebSocket` objects can be iterated, where each iteration yields a message until the connection is closed. """ function receive(ws::WebSocket) @debugv 2 "$(ws.id): Reading message" @require !ws.readclosed frame = readframe(ws.io, Frame, ws.readbuffer) @debugv 2 "$(ws.id): Received frame: $frame" done = checkreadframe!(ws, frame) # common case of reading single non-control frame done && return frame.payload opcode = frame.flags.opcode iscontrol(opcode) && return receive(ws) # if we're here, we're reading a fragmented message payload = frame.payload while true frame = readframe(ws.io, Frame, ws.readbuffer, opcode) @debugv 2 "$(ws.id): Received frame: $frame" done = checkreadframe!(ws, frame) if !iscontrol(frame.flags.opcode) payload = _append(payload, frame.payload) @debugv 2 "$(ws.id): payload len = $(length(payload))" end done && break end payload isa String && utf8check(payload) @debugv 2 "Read message: $(payload[1:min(1024, sizeof(payload))])" return payload end """ iterate(ws) Continuously call `receive(ws)` on a `WebSocket` connection, with each iteration yielding a message until the connection is closed. E.g. ```julia for msg in ws # do something with msg end ``` """ function Base.iterate(ws::WebSocket, st=nothing) isclosed(ws) && return nothing try return receive(ws), nothing catch e isok(e) && return nothing rethrow(e) end end end # module WebSockets X0]+*