Powerfull http-client for Crystal based on libcurl binding. LibCurl is much more powerfull than std crystal http-client, with features like: redirects, interface binding, socks, proxing, sessions, and many others. LibCurl is usually hard to use directly in event based languages like Crystal, because it block main thread. To avoid thread blocking you can use run_with_fork. LibCurl also have multi interface for concurrency, but in my expirience it quite bad and not managable (run_with_fork gives me much more concurrency).
$ brew install curl
$ apt-get install libcurl-dev
Add this to your application's shard.yml
:
dependencies:
curl-downloader:
github: kostya/curl-downloader
require "curl-downloader"
d = Curl::Downloader.new
d.url = "https://google.com/"
d.follow_redirects!
d.headers = {"User-Agent" => "Opera 9.51"}
d.timeout = 60
d.cookie_jar = "/tmp/downloader_test.txt"
d.cookie_file = "/tmp/downloader_test.txt"
d.execute
p d.code
p d.http_status
p d.content[0..100] + "..."
p d.headers.split("\r\n")
p d.url_effective
p d.content_type
d.free
require "curl-downloader"
require "run_with_fork"
require "msgpack"
struct Response
MessagePack.mapping({
code: Int32,
http_status: Int32,
content: String,
})
end
pid, r = Process.run_with_fork do |w|
d = Curl::Downloader.new
d.url = "https://google.com/"
d.follow_redirects!
d.headers = {"User-Agent" => "Opera 9.51"}
d.timeout = 60
d.cookie_jar = "/tmp/downloader_test.txt"
d.cookie_file = "/tmp/downloader_test.txt"
d.execute
{code: d.code,
http_status: d.http_status,
content: d.content}.to_msgpack(w)
end
resp = Response.from_msgpack(r)
p resp
require "curl-downloader"
require "run_with_fork"
def request(url)
d = Curl::Downloader.new
d.url = url
d
end
# run with examples/test-server.cr
reqs = Array.new((ARGV[0]? || 10).to_i) { request("http://127.0.0.1:8089/delay?n=#{rand(1.0)}") }
t = Time.now
ch = Channel(String).new
reqs.each do |req|
spawn do
pid, r = Process.run_with_fork do |w|
req.execute
w.puts(req.url_effective)
end
ch.send r.gets.not_nil!
r.close
end
end
reqs.size.times { puts ch.receive }
p Time.now - t