Compare commits

..

No commits in common. "aa5474167f048fcc91764296efd147ef9e30dbe3" and "177bd1e5721006b3890dd3500c19520486188be1" have entirely different histories.

18 changed files with 202 additions and 244 deletions

View file

@ -1,28 +0,0 @@
name: format
on:
push:
branches:
- main
pull_request:
jobs:
format:
name: format
strategy:
matrix:
ocaml-compiler:
- '5.3'
runs-on: 'ubuntu-latest'
steps:
- uses: actions/checkout@main
- name: Use OCaml ${{ matrix.ocaml-compiler }}
uses: ocaml/setup-ocaml@v3
with:
ocaml-compiler: ${{ matrix.ocaml-compiler }}
dune-cache: true
allow-prerelease-opam: true
- run: opam install ocamlformat.0.27.0
- run: opam exec -- make format-check

View file

@ -3,15 +3,21 @@ name: github pages
on: on:
push: push:
branches: branches:
- main - master # Set a branch name to trigger deployment
jobs: jobs:
deploy: deploy:
runs-on: ubuntu-latest runs-on: ubuntu-latest
permissions:
contents: write
steps: steps:
- uses: actions/checkout@main - uses: actions/checkout@main
- name: Cache opam
id: cache-opam
uses: actions/cache@v2
with:
path: ~/.opam
key: opam-ubuntu-latest-4.12.0
- uses: ocaml/setup-ocaml@v3 - uses: ocaml/setup-ocaml@v3
with: with:
ocaml-compiler: '5.03' ocaml-compiler: '5.03'
@ -33,5 +39,5 @@ jobs:
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./_build/default/_doc/_html/ publish_dir: ./_build/default/_doc/_html/
destination_dir: . destination_dir: dev
enable_jekyll: false enable_jekyll: true

View file

@ -1,4 +1,4 @@
version = 0.27.0 version = 0.26.2
profile=conventional profile=conventional
margin=80 margin=80
if-then-else=k-r if-then-else=k-r

View file

@ -1,38 +1,27 @@
## 0.2.4
# 0.3
- Provide seek function when uploading from string
- make `CURLOPT_NOSIGNAL=false` the default; simplify
- Expose underlying no signal in a global setting
- implement `http_stream`
- feat: add `Ezcurl.Cookies` module, get/set/transfer them
- breaking: wrap Curl.t in record
# 0.2.4
- fix: global initialization logic is now hidden behind a mutex - fix: global initialization logic is now hidden behind a mutex
* depend on `thread` * depend on `thread`
# 0.2.3 ## 0.2.3
- fix: workaround servers which do not understand "Expect" header - fix: workaround servers which do not understand "Expect" header
- fix: correctly set size of payload for POST - fix: correctly set size of payload for POST
- make sure to setup 'PUT" correctly - make sure to setup 'PUT" correctly
- allow POST with non-form data - allow POST with non-form data
# 0.2.2 ## 0.2.2
- fix: do not reset client if passed as argument - fix: do not reset client if passed as argument
# 0.2.1 ## 0.2.1
- fix setting of headers - fix setting of headers
# 0.2 ## 0.2
- add default user agent - add default user agent
# 0.1 ## 0.1
- initial release - initial release

View file

@ -13,12 +13,6 @@ test-autopromote:
clean: clean:
@dune clean @dune clean
format:
@dune build $(DUNE_OPTS) @fmt --auto-promote
format-check:
@dune build $(DUNE_OPTS) @fmt --display=quiet
doc: doc:
@dune build @doc @dune build @doc

11
dune
View file

@ -1,9 +1,10 @@
(rule (rule
(alias runtest) (alias runtest)
(package ezcurl-lwt) (package ezcurl-lwt)
(deps (deps (:file README.md))
(:file README.md))
(action (action
(progn (progn
(run ocaml-mdx test %{deps}) (run ocaml-mdx test %{deps})
(diff? %{file} %{file}.corrected)))) (diff? %{file} %{file}.corrected))))

View file

@ -2,7 +2,7 @@
(name ezcurl) (name ezcurl)
(version 0.3) (version 0.2.4)
(generate_opam_files true) (generate_opam_files true)

View file

@ -1,11 +1,9 @@
open Lwt.Infix open Lwt.Infix
module Str_set = CCSet.Make (String) module Str_set = CCSet.Make(String)
module Uri_tbl = CCHashtbl.Make(struct
module Uri_tbl = CCHashtbl.Make (struct include Uri
include Uri let hash u = Hashtbl.hash (to_string u)
end)
let hash u = Hashtbl.hash (to_string u)
end)
let verbose_ = ref 0 let verbose_ = ref 0
@ -21,7 +19,7 @@ module Run = struct
j: int; j: int;
} }
let push_task (self : t) u : unit = let push_task (self:t) u : unit =
let u = Uri.canonicalize u in let u = Uri.canonicalize u in
if not @@ Uri_tbl.mem self.seen u then ( if not @@ Uri_tbl.mem self.seen u then (
Uri_tbl.add self.seen u (); Uri_tbl.add self.seen u ();
@ -33,109 +31,98 @@ module Run = struct
(* include the domains of [start] in [domains] *) (* include the domains of [start] in [domains] *)
let domains = let domains =
List.fold_left List.fold_left
(fun set uri -> (fun set uri -> match Uri.host uri with
match Uri.host uri with | None -> set
| None -> set | Some h -> Str_set.add h set)
| Some h -> Str_set.add h set)
domains start domains start
in in
let self = let self = {
{ domains; j; max; tasks; default_host; seen=Uri_tbl.create 256;
domains; bad=[]; n=0;
j; } in
max;
tasks;
default_host;
seen = Uri_tbl.create 256;
bad = [];
n = 0;
}
in
List.iter (fun uri -> push_task self uri) start; List.iter (fun uri -> push_task self uri) start;
self self
let bad_code c = c >= 400 let bad_code c = c >= 400
let find_urls (body : string) : Uri.t list = let find_urls (body:string) : Uri.t list =
let body = Soup.parse body in let body = Soup.parse body in
let open Soup.Infix in let open Soup.Infix in
let nodes = body $$ "a[href]" in let nodes = body $$ "a[href]" in
Soup.fold Soup.fold
(fun l n -> (fun l n ->
try try
let url' = Soup.R.attribute "href" n in let url' = Soup.R.attribute "href" n in
Uri.of_string url' :: l Uri.of_string url' :: l
with _ -> l) with _ -> l)
[] nodes [] nodes
let worker (self : t) : unit Lwt.t = let worker (self:t) : unit Lwt.t =
let client = Ezcurl_lwt.make () in let client = Ezcurl_lwt.make () in
let rec loop () = let rec loop() =
if Queue.is_empty self.tasks then if Queue.is_empty self.tasks then Lwt.return ()
Lwt.return () else if self.max >= 0 && self.n > self.max then Lwt.return ()
else if self.max >= 0 && self.n > self.max then
Lwt.return ()
else ( else (
let uri = Queue.pop self.tasks in let uri = Queue.pop self.tasks in
if !verbose_ > 0 then Printf.eprintf "crawl %s\n%!" (Uri.to_string uri); if !verbose_>0 then Printf.eprintf "crawl %s\n%!" (Uri.to_string uri);
(* fetch URL (only 500kb) *) (* fetch URL (only 500kb) *)
self.n <- 1 + self.n; self.n <- 1 + self.n;
Ezcurl_lwt.get ~client ~range:"0-500000" ~url:(Uri.to_string uri) () Ezcurl_lwt.get ~client ~range:"0-500000"~url:(Uri.to_string uri) ()
>>= fun resp -> >>= fun resp ->
(match resp with begin match resp with
| Ok { Ezcurl_lwt.code; body; _ } -> | Ok {Ezcurl_lwt.code; body; _} ->
if bad_code code then ( if bad_code code then (
if !verbose_ > 1 then if !verbose_>1 then (
Printf.eprintf "bad code when fetching %s: %d\n%!" Printf.eprintf "bad code when fetching %s: %d\n%!" (Uri.to_string uri) code;
(Uri.to_string uri) code; );
self.bad <- uri :: self.bad (* bad URL! *) self.bad <- uri :: self.bad; (* bad URL! *)
) else ( ) else (
(* if !verbose_ then Printf.eprintf "body for %s:\n%s\n" (Uri.to_string uri) body; *) (* if !verbose_ then Printf.eprintf "body for %s:\n%s\n" (Uri.to_string uri) body; *)
let cur_host = let cur_host = Uri.host_with_default ~default:self.default_host uri in
Uri.host_with_default ~default:self.default_host uri let uris = find_urls body in
in List.iter
let uris = find_urls body in (fun uri' ->
List.iter match Uri.host uri' with
(fun uri' -> | Some h when Str_set.mem h self.domains ->
match Uri.host uri' with (* follow this link *)
| Some h when Str_set.mem h self.domains -> if !verbose_>1 then Printf.eprintf "follow link to %s\n%!" (Uri.to_string uri');
(* follow this link *) push_task self uri'
if !verbose_ > 1 then | Some _ -> ()
Printf.eprintf "follow link to %s\n%!" (Uri.to_string uri'); | None ->
push_task self uri' (* relative URL, make it absolute *)
| Some _ -> () let uri' = Uri.with_host uri' (Some cur_host) in
| None -> let uri' = Uri.with_scheme uri' (Uri.scheme uri) in
(* relative URL, make it absolute *) if !verbose_>1 then Printf.eprintf "follow link to %s\n%!" (Uri.to_string uri');
let uri' = Uri.with_host uri' (Some cur_host) in push_task self uri'
let uri' = Uri.with_scheme uri' (Uri.scheme uri) in )
if !verbose_ > 1 then uris;
Printf.eprintf "follow link to %s\n%!" (Uri.to_string uri');
push_task self uri') );
uris Lwt.return ()
); | Error (_, msg) ->
Lwt.return () if !verbose_>2 then (
| Error (_, msg) -> Printf.eprintf "error when fetching %s:\n %s\n%!" (Uri.to_string uri) msg;
if !verbose_ > 2 then );
Printf.eprintf "error when fetching %s:\n %s\n%!" self.bad <- uri :: self.bad; (* bad URL! *)
(Uri.to_string uri) msg; Lwt.return ()
self.bad <- uri :: self.bad; end
(* bad URL! *)
Lwt.return ())
>>= loop (* recurse *) >>= loop (* recurse *)
) )
in in
loop () loop()
let run (self : t) : _ Lwt.t = let run (self:t) : _ Lwt.t =
Printf.printf "run %d jobs…\ndomain(s): [%s]\n%!" self.j Printf.printf "run %d jobs…\ndomain(s): [%s]\n%!" self.j
(String.concat "," @@ Str_set.elements self.domains); (String.concat "," @@ Str_set.elements self.domains);
let workers = CCList.init self.j (fun _ -> worker self) in let workers = CCList.init self.j (fun _ -> worker self) in
(* wait for all workers to be done *) (* wait for all workers to be done *)
Lwt.join workers >|= fun () -> self.bad, self.n, Queue.length self.tasks Lwt.join workers >|= fun () ->
self.bad, self.n, Queue.length self.tasks
end end
let help_str = let help_str =
{|A web crawler that can typically be found in Texas. {|A web crawler that can typically be found in Texas.
usage: argiope url [url*] [option*] usage: argiope url [url*] [option*]
|} |}
@ -145,41 +132,34 @@ let () =
let start = ref [] in let start = ref [] in
let j = ref 20 in let j = ref 20 in
let max_ = ref ~-1 in let max_ = ref ~-1 in
let opts = let opts = [
[ "-v", Arg.Unit (fun _ -> incr verbose_), " verbose";
"-v", Arg.Unit (fun _ -> incr verbose_), " verbose"; "--domain", Arg.String (fun s -> domains := Str_set.add s !domains), " include given domain";
( "--domain", "-d", Arg.String (fun s -> domains := Str_set.add s !domains), " alias to --domainm";
Arg.String (fun s -> domains := Str_set.add s !domains), "--max", Arg.Set_int max_, " max number of pages to explore";
" include given domain" ); "-j", Arg.Set_int j, " number of jobs (default 20)";
( "-d", ] |> Arg.align in
Arg.String (fun s -> domains := Str_set.add s !domains),
" alias to --domainm" );
"--max", Arg.Set_int max_, " max number of pages to explore";
"-j", Arg.Set_int j, " number of jobs (default 20)";
]
|> Arg.align
in
Arg.parse opts (CCList.Ref.push start) help_str; Arg.parse opts (CCList.Ref.push start) help_str;
if !start = [] then if !start = [] then (
Arg.usage opts help_str Arg.usage opts help_str;
else ( ) else (
let start = List.map Uri.of_string !start in let start = List.map Uri.of_string !start in
let default_host = let default_host = match Uri.host @@ List.hd start with
match Uri.host @@ List.hd start with
| Some h -> h | Some h -> h
| _ -> failwith "need absolute URIs" | _ -> failwith "need absolute URIs"
| exception _ -> failwith "need absolute URIs" | exception _ -> failwith "need absolute URIs"
in in
let run = Run.make ~default_host ~j:!j ~domains:!domains ~max:!max_ start in let run =
Run.make ~default_host ~j:!j ~domains:!domains ~max:!max_ start
in
(* crawl *) (* crawl *)
let bad, num, remaining = Lwt_main.run (Run.run run) in let bad, num, remaining = Lwt_main.run (Run.run run) in
if bad <> [] then ( if bad <> [] then (
Printf.printf "ERROR: crawled %d pages, %d dead links (%d remaining)\n" Printf.printf "ERROR: crawled %d pages, %d dead links (%d remaining)\n"
num (List.length bad) remaining; num (List.length bad) remaining;
List.iter List.iter (fun uri -> Printf.printf " dead: %s\n" (Uri.to_string uri)) bad;
(fun uri -> Printf.printf " dead: %s\n" (Uri.to_string uri))
bad;
exit 1 exit 1
) else ) else (
Printf.printf "OK: crawled %d pages (remaining %d)\n" num remaining Printf.printf "OK: crawled %d pages (remaining %d)\n" num remaining
)
) )

View file

@ -1,4 +1,5 @@
(executable (executable
(name argiope) (name argiope)
(modes native) (modes native)
(libraries uri lambdasoup lwt ezcurl-lwt containers)) (libraries uri lambdasoup lwt ezcurl-lwt containers))

View file

@ -1,6 +1,6 @@
# This file is generated by dune, edit dune-project instead # This file is generated by dune, edit dune-project instead
opam-version: "2.0" opam-version: "2.0"
version: "0.3" version: "0.2.4"
synopsis: "Friendly wrapper around OCurl, Lwt version" synopsis: "Friendly wrapper around OCurl, Lwt version"
maintainer: ["simon.cruanes.2007@m4x.org"] maintainer: ["simon.cruanes.2007@m4x.org"]
authors: ["Simon Cruanes"] authors: ["Simon Cruanes"]

View file

@ -1,6 +1,6 @@
# This file is generated by dune, edit dune-project instead # This file is generated by dune, edit dune-project instead
opam-version: "2.0" opam-version: "2.0"
version: "0.3" version: "0.2.4"
synopsis: "Friendly wrapper around OCurl" synopsis: "Friendly wrapper around OCurl"
maintainer: ["simon.cruanes.2007@m4x.org"] maintainer: ["simon.cruanes.2007@m4x.org"]
authors: ["Simon Cruanes"] authors: ["Simon Cruanes"]

View file

@ -1,3 +1,4 @@
(library (library
(name ezcurl_core) (name ezcurl_core)
(public_name ezcurl.core) (public_name ezcurl.core)

View file

@ -246,22 +246,23 @@ module type S = sig
@param meth which method to use (see {!meth}) @param meth which method to use (see {!meth})
@param tries how many times to retry in case of [CURLE_AGAIN] code @param tries how many times to retry in case of [CURLE_AGAIN] code
@param client a client to reuse (instead of allocating a new one) @param client a client to reuse (instead of allocating a new one)
@param range @param range an optional
an optional {{: https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests} byte range}
{{:https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests} to fetch (either to get large pages
byte range} to fetch (either to get large pages by chunks, or to resume by chunks, or to resume an interrupted download).
an interrupted download).
@param config configuration to set @param config configuration to set
@param content @param content the content to send as the query's body, either
the content to send as the query's body, either a [`String s] to write a a [`String s] to write a single string, or [`Write f]
single string, or [`Write f] where [f] is a callback that is called on a where [f] is a callback that is called on a buffer [b] with len [n]
buffer [b] with len [n] (as in [f b n]) and returns how many bytes it (as in [f b n]) and returns how many bytes it wrote in the buffer
wrote in the buffer [b] starting at index [0] (at most [n] bytes). It [b] starting at index [0] (at most [n] bytes).
must return [0] when the content is entirely written, and not before. It must return [0] when the content is entirely written, and not
@param headers headers of the query *) before.
@param headers headers of the query
*)
(** Push-stream of bytes (** Push-stream of bytes
@since 0.3 *) @since NEXT_RELEASE *)
class type input_stream = object class type input_stream = object
method on_close : unit -> unit method on_close : unit -> unit
method on_input : bytes -> int -> int -> unit method on_input : bytes -> int -> int -> unit
@ -280,7 +281,7 @@ module type S = sig
unit -> unit ->
(unit response, Curl.curlCode * string) result io (unit response, Curl.curlCode * string) result io
(** HTTP call via cURL, with a streaming response body. (** HTTP call via cURL, with a streaming response body.
@since 0.3 *) @since NEXT_RELEASE *)
val get : val get :
?tries:int -> ?tries:int ->
@ -291,7 +292,9 @@ module type S = sig
url:string -> url:string ->
unit -> unit ->
(string response, Curl.curlCode * string) result io (string response, Curl.curlCode * string) result io
(** Shortcut for [http ~meth:GET] See {!http} for more info. *) (** Shortcut for [http ~meth:GET]
See {!http} for more info.
*)
val put : val put :
?tries:int -> ?tries:int ->
@ -302,7 +305,9 @@ module type S = sig
content:[ `String of string | `Write of bytes -> int -> int ] -> content:[ `String of string | `Write of bytes -> int -> int ] ->
unit -> unit ->
(string response, Curl.curlCode * string) result io (string response, Curl.curlCode * string) result io
(** Shortcut for [http ~meth:PUT] See {!http} for more info. *) (** Shortcut for [http ~meth:PUT]
See {!http} for more info.
*)
val post : val post :
?tries:int -> ?tries:int ->
@ -314,7 +319,9 @@ module type S = sig
url:string -> url:string ->
unit -> unit ->
(string response, Curl.curlCode * string) result io (string response, Curl.curlCode * string) result io
(** Shortcut for [http ~meth:(POST params)] See {!http} for more info. *) (** Shortcut for [http ~meth:(POST params)]
See {!http} for more info.
*)
end end
exception Parse_error of Curl.curlCode * string exception Parse_error of Curl.curlCode * string
@ -361,21 +368,22 @@ module Make (IO : IO) : S with type 'a io = 'a IO.t = struct
n := !n + len; n := !n + len;
r r
and seek i o = and seek i o =
(match o with begin match o with
| Curl.SEEK_SET -> n := Int64.to_int i | Curl.SEEK_SET -> n := Int64.to_int i
| SEEK_END -> n := String.length s + Int64.to_int i | SEEK_END -> n := String.length s + Int64.to_int i
| SEEK_CUR -> n := !n + Int64.to_int i); | SEEK_CUR -> n := !n + Int64.to_int i
end;
Curl.SEEKFUNC_OK Curl.SEEKFUNC_OK
in in read, seek
read, seek
| `Write f -> | `Write f ->
let buf = Bytes.create 1024 in let buf = Bytes.create 1024 in
let read i = let read i =
let len = min i (Bytes.length buf) in let len = min i (Bytes.length buf) in
let n = f buf len in let n = f buf len in
Bytes.sub_string buf i n Bytes.sub_string buf i n
and seek _ _ = Curl.SEEKFUNC_CANTSEEK in and seek _ _ =
read, seek Curl.SEEKFUNC_CANTSEEK
in read, seek
let content_size_ = function let content_size_ = function
| `String s -> Some (String.length s) | `String s -> Some (String.length s)

View file

@ -16,8 +16,8 @@ module Config : sig
end end
type t = private { curl: Curl.t } [@@unboxed] type t = private { curl: Curl.t } [@@unboxed]
(** A client, i.e. a cURL instance. The wrapping record has been present since (** A client, i.e. a cURL instance.
0.3 *) The wrapping record has been present since NEXT_RELEASE *)
val make : val make :
?set_opts:(Curl.t -> unit) -> ?set_opts:(Curl.t -> unit) ->
@ -26,13 +26,10 @@ val make :
unit -> unit ->
t t
(** Create a new client. (** Create a new client.
@param set_opts called before returning the client, to set options @param set_opts called before returning the client, to set options
@param cookiejar_file @param cookiejar_file if provided, tell curl to use the given file path to store/load cookies (since NEXT_RELEASE)
if provided, tell curl to use the given file path to store/load cookies @param enable_session_cookies if provided, enable cookie handling in curl so it store/load cookies (since NEXT_RELEASE)
(since 0.3) *)
@param enable_session_cookies
if provided, enable cookie handling in curl so it store/load cookies
(since 0.3) *)
val delete : t -> unit val delete : t -> unit
(** Delete the client. It cannot be used anymore. *) (** Delete the client. It cannot be used anymore. *)
@ -43,21 +40,21 @@ val with_client : ?set_opts:(Curl.t -> unit) -> (t -> 'a) -> 'a
val set_no_signal : bool -> unit val set_no_signal : bool -> unit
(** Set no_signal default value for each new client instance. Default is [true]. (** Set no_signal default value for each new client instance. Default is [true].
See [CURLOPT_NOSIGNAL]. See [CURLOPT_NOSIGNAL].
@since 0.3 *) @since NEXT_RELEASE *)
(** Cookie handling. (** Cookie handling.
@since 0.3 *) @since NEXT_RELEASE *)
module Cookies : sig module Cookies : sig
val flush_cookiejar : t -> unit val flush_cookiejar : t -> unit
(** If [cookiejar_file] was provided in {!make}, this flushes the current set (** If [cookiejar_file] was provided in {!make}, this flushes the current set of cookies
of cookies to the provided file. to the provided file.
@since 0.3 *) @since NEXT_RELEASE *)
val reload_cookiejar : t -> unit val reload_cookiejar : t -> unit
(** If [cookiejar_file] was provided in {!make}, this reloads cookies from the (** If [cookiejar_file] was provided in {!make}, this reloads cookies from
provided file. the provided file.
@since 0.3 *) @since NEXT_RELEASE *)
val get_cookies : t -> string list val get_cookies : t -> string list
(** Get cookie list (in netscape format) *) (** Get cookie list (in netscape format) *)
@ -75,10 +72,11 @@ end
type response_info = { type response_info = {
ri_response_time: float; ri_response_time: float;
(** Total time (in seconds) for the request/response pair. See (** Total time (in seconds) for the request/response pair.
{!Curl.get_totaltime}. *) See {!Curl.get_totaltime}. *)
ri_redirect_count: int; ri_redirect_count: int;
(** Number of redirects cURL followed. See {!Curl.get_redirectcount}. *) (** Number of redirects cURL followed.
See {!Curl.get_redirectcount}. *)
} }
(** Metadata about a response from the server. *) (** Metadata about a response from the server. *)
@ -87,8 +85,7 @@ val string_of_response_info : response_info -> string
type 'body response = { type 'body response = {
code: int; code: int;
(** Response code. See (** Response code. See https://developer.mozilla.org/en-US/docs/Web/HTTP/Status *)
https://developer.mozilla.org/en-US/docs/Web/HTTP/Status *)
headers: (string * string) list; (** Response headers *) headers: (string * string) list; (** Response headers *)
body: 'body; (** Response body, or [""] *) body: 'body; (** Response body, or [""] *)
info: response_info; (** Information about the response *) info: response_info; (** Information about the response *)
@ -101,9 +98,8 @@ val pp_response_with :
val pp_response : Format.formatter -> string response -> unit val pp_response : Format.formatter -> string response -> unit
val string_of_response : string response -> string val string_of_response : string response -> string
(** The (** The {{: https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods} HTTP method}
{{:https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods} HTTP method} to use *)
to use *)
type meth = type meth =
| GET | GET
| POST of Curl.curlHTTPPost list | POST of Curl.curlHTTPPost list
@ -149,22 +145,23 @@ module type S = sig
@param meth which method to use (see {!meth}) @param meth which method to use (see {!meth})
@param tries how many times to retry in case of [CURLE_AGAIN] code @param tries how many times to retry in case of [CURLE_AGAIN] code
@param client a client to reuse (instead of allocating a new one) @param client a client to reuse (instead of allocating a new one)
@param range @param range an optional
an optional {{: https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests} byte range}
{{:https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests} to fetch (either to get large pages
byte range} to fetch (either to get large pages by chunks, or to resume by chunks, or to resume an interrupted download).
an interrupted download).
@param config configuration to set @param config configuration to set
@param content @param content the content to send as the query's body, either
the content to send as the query's body, either a [`String s] to write a a [`String s] to write a single string, or [`Write f]
single string, or [`Write f] where [f] is a callback that is called on a where [f] is a callback that is called on a buffer [b] with len [n]
buffer [b] with len [n] (as in [f b n]) and returns how many bytes it (as in [f b n]) and returns how many bytes it wrote in the buffer
wrote in the buffer [b] starting at index [0] (at most [n] bytes). It [b] starting at index [0] (at most [n] bytes).
must return [0] when the content is entirely written, and not before. It must return [0] when the content is entirely written, and not
@param headers headers of the query *) before.
@param headers headers of the query
*)
(** Push-based stream of bytes (** Push-based stream of bytes
@since 0.3 *) @since NEXT_RELEASE *)
class type input_stream = object class type input_stream = object
method on_close : unit -> unit method on_close : unit -> unit
method on_input : bytes -> int -> int -> unit method on_input : bytes -> int -> int -> unit
@ -182,10 +179,11 @@ module type S = sig
write_into:#input_stream -> write_into:#input_stream ->
unit -> unit ->
(unit response, Curl.curlCode * string) result io (unit response, Curl.curlCode * string) result io
(** HTTP call via cURL, with a streaming response body. The body is given to (** HTTP call via cURL, with a streaming response body.
[write_into] by chunks, then [write_into#on_close ()] is called and the The body is given to [write_into] by chunks,
response is returned. then [write_into#on_close ()] is called
@since 0.3 *) and the response is returned.
@since NEXT_RELEASE *)
val get : val get :
?tries:int -> ?tries:int ->
@ -196,7 +194,9 @@ module type S = sig
url:string -> url:string ->
unit -> unit ->
(string response, Curl.curlCode * string) result io (string response, Curl.curlCode * string) result io
(** Shortcut for [http ~meth:GET] See {!http} for more info. *) (** Shortcut for [http ~meth:GET]
See {!http} for more info.
*)
val put : val put :
?tries:int -> ?tries:int ->
@ -207,7 +207,9 @@ module type S = sig
content:[ `String of string | `Write of bytes -> int -> int ] -> content:[ `String of string | `Write of bytes -> int -> int ] ->
unit -> unit ->
(string response, Curl.curlCode * string) result io (string response, Curl.curlCode * string) result io
(** Shortcut for [http ~meth:PUT] See {!http} for more info. *) (** Shortcut for [http ~meth:PUT]
See {!http} for more info.
*)
val post : val post :
?tries:int -> ?tries:int ->
@ -219,7 +221,9 @@ module type S = sig
url:string -> url:string ->
unit -> unit ->
(string response, Curl.curlCode * string) result io (string response, Curl.curlCode * string) result io
(** Shortcut for [http ~meth:(POST params)] See {!http} for more info. *) (** Shortcut for [http ~meth:(POST params)]
See {!http} for more info.
*)
end end
module Make (IO : IO) : S with type 'a io = 'a IO.t module Make (IO : IO) : S with type 'a io = 'a IO.t

View file

@ -1,3 +1,4 @@
(library (library
(name ezcurl_lwt) (name ezcurl_lwt)
(public_name ezcurl-lwt) (public_name ezcurl-lwt)

View file

@ -16,3 +16,4 @@ include Ezcurl_core.Make (struct
Curl.CURLE_OK Curl.CURLE_OK
with Curl.CurlException (c, _, _) -> c with Curl.CurlException (c, _, _) -> c
end) end)

View file

@ -1,6 +1,6 @@
get: OK get: OK
body=``` body=```
version = 0.27.0 version = 0.26.2
profile=conventional profile=conventional
margin=80 margin=80
if-then-else=k-r if-then-else=k-r
@ -19,7 +19,7 @@ ocaml-version=4.08.0
``` ```
streaming get: OK streaming get: OK
body=``` body=```
version = 0.27.0 version = 0.26.2
profile=conventional profile=conventional
margin=80 margin=80
if-then-else=k-r if-then-else=k-r

View file

@ -1,3 +1,3 @@
(test (test
(name basic_test) (name basic_test)
(libraries ezcurl)) (libraries ezcurl))