git mv Ingress ingress

This commit is contained in:
Prashanth Balasubramanian 2016-02-21 16:13:08 -08:00
parent 34b949c134
commit 3da4e74e5a
2185 changed files with 754743 additions and 0 deletions

View file

@ -0,0 +1,44 @@
local ssl = require "ngx.ssl"
local ssl_base_directory = "/etc/nginx/nginx-ssl"
local server_name = ssl.server_name()
local addr, addrtyp, err = ssl.raw_server_addr()
local byte = string.byte
local cert_path = ""
ssl.clear_certs()
-- Check for SNI request.
if server_name == nil then
ngx.log(ngx.INFO, "SNI Not present - performing IP lookup")
-- Set server name as IP address.
server_name = string.format("%d.%d.%d.%d", byte(addr, 1), byte(addr, 2), byte(addr, 3), byte(addr, 4))
ngx.log(ngx.INFO, "IP Address: ", server_name)
end
-- Set certifcate paths
cert_path = ssl_base_directory .. "/" .. server_name .. ".cert"
key_path = ssl_base_directory .. "/" .. server_name .. ".key"
-- Attempt to retrieve and set certificate for request.
local f = assert(io.open(cert_path))
local cert_data = f:read("*a")
f:close()
local ok, err = ssl.set_der_cert(cert_data)
if not ok then
ngx.log(ngx.ERR, "failed to set DER cert: ", err)
return
end
-- Attempt to retrieve and set key for request.
local f = assert(io.open(key_path))
local pkey_data = f:read("*a")
f:close()
local ok, err = ssl.set_der_priv_key(pkey_data)
if not ok then
ngx.log(ngx.ERR, "failed to set DER key: ", err)
return
end

View file

@ -0,0 +1,50 @@
http = require "resty.http"
function openURL(status, page)
local httpc = http.new()
local res, err = httpc:request_uri(page, {
path = "/",
method = "GET"
})
if not res then
ngx.log(ngx.ERR, err)
ngx.exit(500)
end
ngx.status = tonumber(status)
ngx.header["Content-Type"] = ngx.var.httpReturnType or "text/plain"
if ngx.var.http_cookie then
ngx.header["Cookie"] = ngx.var.http_cookie
end
ngx.say(res.body)
end
function openCustomErrorURL(status, page)
local httpc = http.new()
data = {}
data["code"] = status
data["format"] = ngx.var.httpAccept
local params = "/error?"..ngx.encode_args(data)
local res, err = httpc:request_uri(page, {
path = params,
method = "GET"
})
if not res then
ngx.log(ngx.ERR, err)
ngx.exit(500)
end
ngx.status = tonumber(status)
ngx.header["Content-Type"] = ngx.var.httpReturnType or "text/plain"
if ngx.var.http_cookie then
ngx.header["Cookie"] = ngx.var.http_cookie
end
ngx.say(res.body)
end

View file

@ -0,0 +1,229 @@
local _M = {}
local cjson = require "cjson"
local trie = require "trie"
local http = require "resty.http"
local cache = require "resty.dns.cache"
local os = require "os"
local encode = cjson.encode
local decode = cjson.decode
local table_concat = table.concat
local trie_get = trie.get
local match = string.match
local gsub = string.gsub
local lower = string.lower
-- we "cache" the config local to each worker
local ingressConfig = nil
local cluster_domain = "cluster.local"
local def_backend = nil
local custom_error = nil
local dns_cache_options = nil
function get_ingressConfig(ngx)
local d = ngx.shared["ingress"]
local value, flags, stale = d:get_stale("ingressConfig")
if not value then
-- nothing we can do
return nil, "config not set"
end
ingressConfig = decode(value)
return ingressConfig, nil
end
function worker_cache_config(ngx)
local _, err = get_ingressConfig(ngx)
if err then
ngx.log(ngx.ERR, "unable to get ingressConfig: ", err)
return
end
end
function _M.content(ngx)
local host = ngx.var.host
-- strip off any port
local h = match(host, "^(.+):?")
if h then
host = h
end
host = lower(host)
local config, err = get_ingressConfig(ngx)
if err then
ngx.log(ngx.ERR, "unable to get config: ", err)
return ngx.exit(503)
end
-- this assumes we only allow exact host matches
local paths = config[host]
if not paths then
ngx.log(ngx.ERR, "No server for host "..host.." returning 404")
if custom_error then
openCustomErrorURL(404, custom_error)
return
else
openURL(404, def_backend)
return
end
end
local backend = trie_get(paths, ngx.var.uri)
if not backend then
ngx.log(ngx.ERR, "No server for host "..host.." and path "..ngx.var.uri.." returning 404")
if custom_error then
openCustomErrorURL(404, custom_error)
return
else
openURL(404, def_backend)
return
end
end
local address = backend.host
ngx.var.upstream_port = backend.port or 80
if dns_cache_options then
local dns = cache.new(dns_cache_options)
local answer, err, stale = dns:query(address, { qtype = 1 })
if err or (not answer) then
if stale then
answer = stale
else
answer = nil
end
end
if answer and answer[1] then
local ans = answer[1]
if ans.address then
address = ans.address
end
else
ngx.log(ngx.ERR, "dns failed for ", address, " with ", err, " => ", encode(answer or ""))
end
end
ngx.var.upstream_host = address
return
end
function _M.init_worker(ngx)
end
function _M.init(ngx, options)
-- ngx.log(ngx.OK, "options: "..encode(options))
def_backend = options.def_backend
custom_error = options.custom_error
-- try to create a dns cache
local resolvers = options.resolvers
if resolvers then
cache.init_cache(512)
local servers = trie.strsplit(" ", resolvers)
dns_cache_options =
{
dict = "dns_cache",
negative_ttl = nil,
max_stale = 900,
normalise_ttl = false,
resolver = {
nameservers = {servers[1]}
}
}
end
end
-- dump config. This is the raw config (including trie) for now
function _M.config(ngx)
ngx.header.content_type = "application/json"
local config = {
ingress = ingressConfig
}
local val = encode(config)
ngx.print(val)
end
function _M.update_ingress(ngx)
ngx.header.content_type = "application/json"
if ngx.req.get_method() ~= "POST" then
ngx.print(encode({
message = "only POST request"
}))
ngx.exit(400)
return
end
ngx.req.read_body()
local data = ngx.req.get_body_data()
local val = decode(data)
if not val then
ngx.log(ngx.ERR, "failed to decode body")
return
end
config = {}
for _, ingress in ipairs(val) do
local namespace = ingress.metadata.namespace
local spec = ingress.spec
-- we do not allow default ingress backends right now.
for _, rule in ipairs(spec.rules) do
local host = rule.host
local paths = config[host]
if not paths then
paths = trie.new()
config[host] = paths
end
rule.http = rule.http or { paths = {}}
for _, path in ipairs(rule.http.paths) do
local hostname = table_concat(
{
path.backend.serviceName,
namespace,
"svc",
cluster_domain
}, ".")
local backend = {
host = hostname,
port = path.backend.servicePort
}
paths:add(path.path, backend)
end
end
end
local d = ngx.shared["ingress"]
local ok, err, _ = d:set("ingressConfig", encode(config))
if not ok then
ngx.log(ngx.ERR, "Error: "..err)
local res = encode({
message = "Error updating Ingress rules: "..err
})
ngx.print(res)
return ngx.exit(500)
end
ingressConfig = config
local res = encode({
message = "Ingress rules updated"
})
ngx.print(res)
end
return _M

View file

@ -0,0 +1,78 @@
-- Simple trie for URLs
local _M = {}
local mt = {
__index = _M
}
-- http://lua-users.org/wiki/SplitJoin
local strfind, tinsert, strsub = string.find, table.insert, string.sub
function _M.strsplit(delimiter, text)
local list = {}
local pos = 1
while 1 do
local first, last = strfind(text, delimiter, pos)
if first then -- found?
tinsert(list, strsub(text, pos, first-1))
pos = last+1
else
tinsert(list, strsub(text, pos))
break
end
end
return list
end
local strsplit = _M.strsplit
function _M.new()
local t = { }
return setmetatable(t, mt)
end
function _M.add(t, key, val)
local parts = {}
-- hack for just /
if key == "/" then
parts = { "" }
else
parts = strsplit("/", key)
end
local l = t
for i = 1, #parts do
local p = parts[i]
if not l[p] then
l[p] = {}
end
l = l[p]
end
l.__value = val
end
function _M.get(t, key)
local parts = strsplit("/", key)
local l = t
-- this may be nil
local val = t.__value
for i = 1, #parts do
local p = parts[i]
if l[p] then
l = l[p]
local v = l.__value
if v then
val = v
end
else
break
end
end
-- may be nil
return val
end
return _M

View file

@ -0,0 +1,2 @@
t/servroot/
t/error.log

View file

@ -0,0 +1,20 @@
The MIT License (MIT)
Copyright (c) 2013 Hamish Forbes
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View file

@ -0,0 +1,23 @@
OPENRESTY_PREFIX=/usr/local/openresty
PREFIX ?= /usr/local
LUA_INCLUDE_DIR ?= $(PREFIX)/include
LUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION)
INSTALL ?= install
TEST_FILE ?= t
.PHONY: all test leak
all: ;
install: all
$(INSTALL) -d $(DESTDIR)/$(LUA_LIB_DIR)/resty/dns
leak: all
TEST_NGINX_CHECK_LEAK=1 TEST_NGINX_NO_SHUFFLE=1 PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$$PATH prove -I../test-nginx/lib -r $(TEST_FILE)
test: all
TEST_NGINX_NO_SHUFFLE=1 PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$$PATH prove -I../test-nginx/lib -r $(TEST_FILE)
util/lua-releng.pl

View file

@ -0,0 +1,111 @@
#lua-resty-dns-cache
A wrapper for [lua-resty-dns](https://github.com/openresty/lua-resty-dns) to cache responses based on record TTLs.
Uses [lua-resty-lrucache](https://github.com/openresty/lua-resty-lrucache) and [ngx.shared.DICT](https://github.com/openresty/lua-nginx-module#ngxshareddict) to provide a 2 level cache.
Can repopulate cache in the background while returning stale answers.
#Overview
```lua
lua_shared_dict dns_cache 1m;
init_by_lua '
require("resty.dns.cache").init_cache(200)
';
server {
listen 80;
server_name dns_cache;
location / {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns = DNS_Cache.new({
dict = "dns_cache",
negative_ttl = 30,
max_stale = 300,
resolver = {
nameservers = {"123.123.123.123"}
}
})
local host = ngx.req.get_uri_args()["host"] or "www.google.com"
local answer, err, stale = dns:query(host)
if err then
if stale then
ngx.header["Warning"] = "110: Response is stale"
answer = stale
ngx.log(ngx.ERR, err)
else
ngx.status = 500
ngx.say(err)
return ngx.exit(ngx.status)
end
end
local cjson = require "cjson"
ngx.say(cjson.encode(answer))
';
}
}
```
#Methods
### init_cache
`syntax: ok, err = dns_cache.init_cache(max_items?)`
Creates a global lrucache object for caching responses.
Accepts an optional `max_items` argument, defaults to 200 entries.
Calling this repeatedly will reset the LRU cache
### initted
`syntax: ok = dns_cache.initted()`
Returns `true` if LRU Cache has been initialised
### new
`syntax: ok, err = dns_cache.new(opts)`
Returns a new DNS cache instance. Returns `nil` and a string on error
Accepts a table of options, if no shared dictionary is provided only lrucache is used.
* `dict` - Name of the [ngx.shared.DICT](https://github.com/openresty/lua-nginx-module#ngxshareddict) to use for cache.
* `resolver` - Table of options passed to [lua-resty-dns](https://github.com/openresty/lua-resty-dns#new). Defaults to using Google DNS.
* `normalise_ttl` - Boolean. Reduces TTL in cached answers to account for cached time. Defaults to `true`.
* `negative_ttl` - Time in seconds to cache negative / error responses. `nil` or `false` disables caching negative responses. Defaults to `false`
* `minimise_ttl` - Boolean. Set cache TTL based on the shortest DNS TTL in all responses rather than the first response. Defaults to `false`
* `max_stale` - Number of seconds past expiry to return stale content rather than querying. Stale hits will trigger a non-blocking background query to repopulate cache.
### query
`syntax: answers, err, stale = c:query(name, opts?)`
Passes through to lua-resty-dns' [query](https://github.com/openresty/lua-resty-dns#query) method.
Returns an extra `stale` variable containing stale data if a resolver cannot be contacted.
### tcp_query
`syntax: answers, err, stale = c:tcp_query(name, opts?)`
Passes through to lua-resty-dns' [tcp_query](https://github.com/openresty/lua-resty-dns#tcp_query) method.
Returns an extra `stale` variable containing stale data if a resolver cannot be contacted.
### set_timeout
`syntax: c:set_timeout(time)`
Passes through to lua-resty-dns' [set_timeout](https://github.com/openresty/lua-resty-dns#set_timeout) method.
## Constants
lua-resty-dns' [constants](https://github.com/openresty/lua-resty-dns#constants) are accessible on the `resty.dns.cache` object too.
## TODO
* Cap'n'proto serialisation

View file

@ -0,0 +1,449 @@
local ngx_log = ngx.log
local ngx_DEBUG = ngx.DEBUG
local ngx_ERR = ngx.ERR
local ngx_shared = ngx.shared
local ngx_time = ngx.time
local resty_resolver = require "resty.dns.resolver"
local resty_lrucache = require "resty.lrucache"
local cjson = require "cjson"
local json_encode = cjson.encode
local json_decode = cjson.decode
local tbl_concat = table.concat
local tonumber = tonumber
local _ngx_timer_at = ngx.timer.at
local ngx_worker_pid = ngx.worker.pid
local function ngx_timer_at(delay, func, ...)
local ok, err = _ngx_timer_at(delay, func, ...)
if not ok then
ngx_log(ngx_ERR, "Timer Error: ", err)
end
return ok, err
end
local debug_log = function(msg, ...)
if type(msg) == 'table' then
local ok, json = pcall(json_encode, msg)
if ok then
msg = json
else
ngx_log(ngx_ERR, json)
end
end
ngx_log(ngx_DEBUG, msg, ...)
end
local _M = {
_VERSION = '0.01',
TYPE_A = resty_resolver.TYPE_A,
TYPE_NS = resty_resolver.TYPE_NS,
TYPE_CNAME = resty_resolver.TYPE_CNAME,
TYPE_PTR = resty_resolver.TYPE_PTR,
TYPE_MX = resty_resolver.TYPE_MX,
TYPE_TXT = resty_resolver.TYPE_TXT,
TYPE_AAAA = resty_resolver.TYPE_AAAA,
TYPE_SRV = resty_resolver.TYPE_SRV,
TYPE_SPF = resty_resolver.TYPE_SPF,
CLASS_IN = resty_resolver.CLASS_IN
}
local DEBUG = false
local mt = { __index = _M }
local lru_cache_defaults = {200}
local resolver_defaults = {
nameservers = {"8.8.8.8", "8.8.4.4"}
}
-- Global lrucache instance
local lru_cache
local max_items = 200
function _M.init_cache(size)
if size then max_items = size end
local err
if DEBUG then debug_log("Initialising lru cache with ", max_items, " max items") end
lru_cache, err = resty_lrucache.new(max_items)
if not lru_cache then
return nil, err
end
return true
end
function _M.initted()
if lru_cache then return true end
return false
end
function _M.new(opts)
local self, err = { opts = opts}, nil
opts = opts or {}
-- Set defaults
if opts.normalise_ttl ~= nil then self.normalise_ttl = opts.normalise_ttl else self.normalise_ttl = true end
if opts.minimise_ttl ~= nil then self.minimise_ttl = opts.minimise_ttl else self.minimise_ttl = false end
if opts.negative_ttl ~= nil then
self.negative_ttl = tonumber(opts.negative_ttl)
else
self.negative_ttl = false
end
if opts.max_stale ~= nil then
self.max_stale = tonumber(opts.max_stale)
else
self.max_stale = 0
end
opts.resolver = opts.resolver or resolver_defaults
self.resolver, err = resty_resolver:new(opts.resolver)
if not self.resolver then
return nil, err
end
if opts.dict then
self.dict = ngx_shared[opts.dict]
end
return setmetatable(self, mt)
end
function _M.flush(self, hard)
local ok, err = self.init_cache()
if not ok then
ngx_log(ngx_ERR, err)
end
if self.dict then
if DEBUG then debug_log("Flushing dictionary") end
self.dict:flush_all()
if hard then
local flushed = self.dict:flush_expired()
if DEBUG then debug_log("Flushed ", flushed, " keys from memory") end
end
end
end
function _M._debug(flag)
DEBUG = flag
end
function _M.set_timeout(self, ...)
return self.resolver:set_timeout(...)
end
local function minimise_ttl(answer)
if DEBUG then debug_log('Minimising TTL') end
local ttl
for _, ans in ipairs(answer) do
if DEBUG then debug_log('TTL ', ans.name, ': ', ans.ttl) end
if ttl == nil or ans.ttl < ttl then
ttl = ans.ttl
end
end
return ttl
end
local function normalise_ttl(self, data)
-- Calculate time since query and subtract from answer's TTL
if self.normalise_ttl then
local now = ngx_time()
local diff = now - data.now
if DEBUG then debug_log("Normalising TTL, diff: ", diff) end
for _, answer in ipairs(data.answer) do
if DEBUG then debug_log("Old: ", answer.ttl, ", new: ", answer.ttl - diff) end
answer.ttl = answer.ttl - diff
end
data.now = now
end
return data
end
local function cache_get(self, key)
-- Try local LRU cache first
local data, lru_stale
if lru_cache then
data, lru_stale = lru_cache:get(key)
-- Set stale if should have expired
if data and data.expires <= ngx_time() then
lru_stale = data
data = nil
end
if data then
if DEBUG then
debug_log('lru_cache HIT: ', key)
debug_log(data)
end
return normalise_ttl(self, data)
elseif DEBUG then
debug_log('lru_cache MISS: ', key)
end
end
-- lru_cache miss, try shared dict
local dict = self.dict
if dict then
local data, flags, stale = dict:get_stale(key)
-- Set stale if should have expired
if data then
data = json_decode(data)
if data.expires <= ngx_time() then
stale = true
end
end
-- Dict data is stale, prefer stale LRU data
if stale and lru_stale then
if DEBUG then
debug_log('lru_cache STALE: ', key)
debug_log(lru_stale)
end
return nil, normalise_ttl(self, lru_stale)
end
-- Definitely no lru data, going to have to try shared dict
if not data then
-- Full MISS on dict, return nil
if DEBUG then debug_log('shared_dict MISS: ', key) end
return nil
end
-- Return nil and dict cache if its stale
if stale then
if DEBUG then debug_log('shared_dict STALE: ', key) end
return nil, normalise_ttl(self, data)
end
-- Fresh HIT from dict, repopulate the lru_cache
if DEBUG then debug_log('shared_dict HIT: ', key) end
if lru_cache then
local ttl = data.expires - ngx_time()
if DEBUG then debug_log('lru_cache SET: ', key, ' ', ttl) end
lru_cache:set(key, data, ttl)
end
return normalise_ttl(self, data)
elseif lru_stale then
-- Return lru stale if no dict configured
if DEBUG then
debug_log('lru_cache STALE: ', key)
debug_log(lru_stale)
end
return nil, normalise_ttl(self, lru_stale)
end
if not lru_cache or dict then
ngx_log(ngx_ERR, "No cache defined")
end
end
local function cache_set(self, key, answer, ttl)
-- Don't cache records with 0 TTL
if ttl == 0 or ttl == nil then
return
end
-- Calculate absolute expiry - used to populate lru_cache from shared_dict
local now = ngx_time()
local data = {
answer = answer,
now = now,
queried = now,
expires = now + ttl
}
-- Extend cache expiry if using stale
local real_ttl = ttl
if self.max_stale then
real_ttl = real_ttl + self.max_stale
end
-- Set lru cache
if lru_cache then
if DEBUG then debug_log('lru_cache SET: ', key, ' ', real_ttl) end
lru_cache:set(key, data, real_ttl)
end
-- Set dict cache
local dict = self.dict
if dict then
if DEBUG then debug_log('shared_dict SET: ', key, ' ', real_ttl) end
local ok, err, forcible = dict:set(key, json_encode(data), real_ttl)
if not ok then
ngx_log(ngx_ERR, 'shared_dict ERR: ', err)
end
if forcible then
ngx_log(ngx_DEBUG, 'shared_dict full')
end
end
end
local function _resolve(resolver, query_func, host, opts)
if DEBUG then debug_log('Querying: ', host) end
local answers, err = query_func(resolver, host, opts)
if not answers then
return answers, err
end
if DEBUG then debug_log(answers) end
return answers
end
local function cache_key(host, qtype)
return tbl_concat({host,'|',qtype})
end
local function get_repopulate_lock(dict, host, qtype)
local key = cache_key(host, qtype or 1) .. '|lock'
if DEBUG then debug_log("Locking '", key, "' for ", 30, "s: ", ngx_worker_pid()) end
return dict:add(key, ngx_worker_pid(), 30)
end
local function release_repopulate_lock(dict, host, qtype)
local key = cache_key(host, qtype or 1) .. '|lock'
local pid, err = dict:get(key)
if DEBUG then debug_log("Releasing '", key, "' for ", ngx_worker_pid(), " from ", pid) end
if pid == ngx_worker_pid() then
dict:delete(key)
else
ngx_log(ngx_DEBUG, "couldnt release lock")
end
end
local _query
local function _repopulate(premature, self, host, opts, tcp)
if premature then return end
if DEBUG then debug_log("Repopulating '", host, "'") end
-- Create a new resolver instance, cannot share sockets
local err
self.resolver, err = resty_resolver:new(self.opts.resolver)
if err then
ngx_log(ngx_ERR, err)
return nil
end
-- Do not use stale when repopulating
_query(self, host, opts, tcp, true)
end
local function repopulate(self, host, opts, tcp)
-- Lock, there's a window between the key expiring and the background query completing
-- during which another query could trigger duplicate repopulate jobs
local ok, err = get_repopulate_lock(self.dict, host, opts.qtype)
if ok then
if DEBUG then debug_log("Attempting to repopulate '", host, "'") end
local ok, err = ngx_timer_at(0, _repopulate, self, host, opts, tcp)
if not ok then
-- Release lock if we couldn't start the timer
release_repopulate_lock(self.dict, host, opts.qtype)
end
else
if err == "exists" then
if DEBUG then debug_log("Lock not acquired") end
return
else
ngx.log(ngx.ERR, err)
end
end
end
_query = function(self, host, opts, tcp, repopulating)
-- Build cache key
opts = opts or {}
local key = cache_key(host, opts.qtype or 1)
-- Check caches
local answer
local data, stale = cache_get(self, key)
if data then
-- Shouldn't get a cache hit when repopulating but better safe than sorry
if repopulating then release_repopulate_lock(self.dict, host, opts.qtype) end
answer = data.answer
-- Don't return negative cache hits if negative_ttl is off in this instance
if not answer.errcode or self.negative_ttl then
return answer
end
end
-- No fresh cache entry, return stale if within max_stale and trigger background repopulate
if stale and not repopulating and self.max_stale > 0
and (ngx_time() - stale.expires) < self.max_stale then
if DEBUG then debug_log('max_stale ', self.max_stale) end
repopulate(self, host, opts, tcp)
if DEBUG then debug_log('Returning STALE: ', key) end
return nil, nil, stale.answer
end
-- Try to resolve
local resolver = self.resolver
local query_func = resolver.query
if tcp then
query_func = resolver.tcp_query
end
local answer, err = _resolve(resolver, query_func, host, opts)
if not answer then
-- Couldn't resolve, return potential stale response with error msg
if DEBUG then
debug_log('Resolver error ', key, ': ', err)
if stale then debug_log('Returning STALE: ', key) end
end
if repopulating then release_repopulate_lock(self.dict, host, opts.qtype) end
if stale then stale = stale.answer end
return nil, err, stale
end
local ttl
-- Cache server errors for negative_cache seconds
if answer.errcode then
if self.negative_ttl then
ttl = self.negative_ttl
else
if repopulating then release_repopulate_lock(self.dict, host, opts.qtype) end
return answer
end
else
-- Cache for the lowest TTL in the chain of responses...
if self.minimise_ttl then
ttl = minimise_ttl(answer)
elseif answer[1] then
-- ... or just the first one
ttl = answer[1].ttl or nil
end
end
-- Set cache
cache_set(self, key, answer, ttl)
if repopulating then release_repopulate_lock(self.dict, host, opts.qtype) end
return answer
end
function _M.query(self, host, opts)
return _query(self, host, opts, false)
end
function _M.tcp_query(self, host, opts)
return _query(self, host, opts, true)
end
return _M

View file

@ -0,0 +1,233 @@
use Test::Nginx::Socket;
use Cwd qw(cwd);
plan tests => repeat_each() * 24;
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
};
no_long_string();
run_tests();
__DATA__
=== TEST 1: Load module without errors.
--- http_config eval
"$::HttpConfig"
. q{
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
';
}
--- config
location /sanity {
echo "OK";
}
--- request
GET /sanity
--- no_error_log
[error]
--- response_body
OK
=== TEST 2: Can init cache - defaults
--- http_config eval
"$::HttpConfig"
. q{
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /sanity {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
ngx.say(DNS_Cache.initted())
';
}
--- request
GET /sanity
--- no_error_log
[error]
--- response_body
true
=== TEST 3: Can init cache - user config
--- http_config eval
"$::HttpConfig"
. q{
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache(300)
';
}
--- config
location /sanity {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
ngx.say(DNS_Cache.initted())
';
}
--- request
GET /sanity
--- no_error_log
[error]
--- response_body
true
=== TEST 4: Can init new instance - defaults
--- http_config eval
"$::HttpConfig"
. q{
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache(300)
';
}
--- config
location /sanity {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new()
if dns then
ngx.say("OK")
else
ngx.say(err)
end
';
}
--- request
GET /sanity
--- no_error_log
[error]
--- response_body
OK
=== TEST 5: Can init new instance - user config
--- http_config eval
"$::HttpConfig"
. q{
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache(300)
';
}
--- config
location /sanity {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
negative_ttl = 10,
resolver = { nameservers = {"10.10.10.10"} }
})
if dns then
ngx.say("OK")
else
ngx.say(err)
end
';
}
--- request
GET /sanity
--- no_error_log
[error]
--- response_body
OK
=== TEST 6: Resty DNS errors are passed through
--- http_config eval
"$::HttpConfig"
. q{
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache(300)
';
}
--- config
location /sanity {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
resolver = { }
})
if dns then
ngx.say("OK")
else
ngx.say(err)
end
';
}
--- request
GET /sanity
--- no_error_log
[error]
--- response_body
no nameservers specified
=== TEST 7: Can create instance with shared dict
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /sanity {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
ngx.say(DNS_Cache.initted())
local dns, err = DNS_Cache.new({
dict = "dns_cache"
})
if dns then
ngx.say("OK")
else
ngx.say(err)
end
';
}
--- request
GET /sanity
--- no_error_log
[error]
--- response_body
true
OK
=== TEST 8: Can create instance with shared dict and no lru_cache
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
}
--- config
location /sanity {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
ngx.say(DNS_Cache.initted())
local dns, err = DNS_Cache.new({
dict = "dns_cache"
})
if dns then
ngx.say("OK")
else
ngx.say(err)
end
';
}
--- request
GET /sanity
--- no_error_log
[error]
--- response_body
false
OK

View file

@ -0,0 +1,195 @@
use lib 't';
use TestDNS;
use Cwd qw(cwd);
plan tests => repeat_each() * 12;
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
};
no_long_string();
run_tests();
__DATA__
=== TEST 1: Can resolve with lru + dict
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {
nameservers = { {"127.0.0.1", "1953"} }
}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 123456 }],
}
--- request
GET /t
--- no_error_log
[error]
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":123456}]
=== TEST 2: Can resolve with lru only
--- http_config eval
"$::HttpConfig"
. q{
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
resolver = {
nameservers = { {"127.0.0.1", "1953"} }
}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 123456 }],
}
--- request
GET /t
--- no_error_log
[error]
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":123456}]
=== TEST 3: Can resolve with dict only
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
}
--- config
location /t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {
nameservers = { {"127.0.0.1", "1953"} }
}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 123456 }],
}
--- request
GET /t
--- no_error_log
[error]
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":123456}]
=== TEST 4: Can resolve with no cache, error thrown
--- http_config eval
"$::HttpConfig"
--- config
location /t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
resolver = {
nameservers = { {"127.0.0.1", "1953"} }
}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 123456 }],
}
--- request
GET /t
--- error_log
No cache defined
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":123456}]

View file

@ -0,0 +1,873 @@
use lib 't';
use TestDNS;
use Cwd qw(cwd);
plan tests => repeat_each() * 47;
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
lua_socket_log_errors off;
};
no_long_string();
run_tests();
__DATA__
=== TEST 1: Response comes from cache on second hit
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
echo_location /_t;
echo_location /_t2;
}
location /_t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
';
}
location /_t2 {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
dns._debug(true)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 123456 }],
}
--- request
GET /t
--- error_log
lru_cache HIT
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":123456}]
=== TEST 2: Response comes from dict on miss
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
echo_location /_t;
echo_location /_t2;
}
location /_t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
';
}
location /_t2 {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache() -- reset cache
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
dns._debug(true)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 123456 }],
}
--- request
GET /t
--- error_log
lru_cache MISS
shared_dict HIT
lru_cache SET
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":123456}]
=== TEST 3: Stale response from lru served if resolver down
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
echo_location /_t;
echo_sleep 2;
echo_location /_t2;
}
location /_t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
';
}
location /_t2 {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1954"}}, retrans = 1, timeout = 100}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
dns._debug(true)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if stale then
answer = stale
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 1 }],
}
--- request
GET /t
--- error_log
lru_cache MISS
lru_cache STALE
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":-1}]
=== TEST 4: Stale response from dict served if resolver down
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
echo_location /_t;
echo_sleep 2;
echo_location /_t2;
}
location /_t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
';
}
location /_t2 {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1954"}}, retrans = 1, timeout = 100}
})
DNS_Cache.init_cache() -- reset cache
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
dns._debug(true)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if stale then
answer = stale
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 1 }],
}
--- request
GET /t
--- error_log
lru_cache MISS
shared_dict STALE
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":-1}]
=== TEST 5: Stale response from lru served if resolver down, no dict
--- http_config eval
"$::HttpConfig"
. q{
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
echo_location /_t;
echo_sleep 2;
echo_location /_t2;
}
location /_t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
';
}
location /_t2 {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1954"}}, retrans = 1, timeout = 100}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
dns._debug(true)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if stale then
answer = stale
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 1 }],
}
--- request
GET /t
--- error_log
lru_cache MISS
lru_cache STALE
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":-1}]
=== TEST 6: Stale response from dict served if resolver down, no lru
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
}
--- config
location /t {
echo_location /_t;
echo_sleep 2;
echo_location /_t2;
}
location /_t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
';
}
location /_t2 {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1954"}}, retrans = 1, timeout = 100}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
dns._debug(true)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if stale then
answer = stale
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 1 }],
}
--- request
GET /t
--- error_log
shared_dict STALE
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":-1}]
=== TEST 7: TTLs are reduced
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
echo_location /_t;
echo_sleep 2;
echo_location /_t2;
}
location /_t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
';
}
location /_t2 {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}, retrans = 1, timeout = 100}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(answer)
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 10 }],
}
--- request
GET /t
--- no_error_log
[error]
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":8}]
=== TEST 8: TTL reduction can be disabled
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
echo_location /_t;
echo_sleep 2;
echo_location /_t2;
}
location /_t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
';
}
location /_t2 {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
normalise_ttl = false,
resolver = {nameservers = {{"127.0.0.1", "1953"}}, retrans = 1, timeout = 100}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(answer)
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 10 }],
}
--- request
GET /t
--- no_error_log
[error]
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":10}]
=== TEST 9: Negative responses are not cached by default
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns._debug(true)
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
rcode => 3,
opcode => 0,
qname => 'www.google.com',
}
--- request
GET /t
--- no_error_log
SET
--- response_body
{"errcode":3,"errstr":"name error"}
=== TEST 10: Negative responses can be cached
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
echo_location /_t;
echo_location /_t2;
}
location /_t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
negative_ttl = 10,
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
';
}
location /_t2 {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
negative_ttl = 10,
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
dns._debug(true)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
rcode => 3,
opcode => 0,
qname => 'www.google.com',
}
--- request
GET /t
--- error_log
lru_cache HIT
--- response_body
{"errcode":3,"errstr":"name error"}
=== TEST 11: Cached negative responses are not returned by default
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
echo_location /_t;
echo_location /_t2;
}
location /_t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
negative_ttl = 10,
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns._debug(true)
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
';
}
location /_t2 {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1954"}, retrans = 1, timeout = 100}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
dns._debug(true)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
rcode => 3,
opcode => 0,
qname => 'www.google.com',
}
--- request
GET /t
--- error_log
lru_cache SET
lru_cache HIT
--- response_body
null
=== TEST 12: Cache TTL can be minimised
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
minimise_ttl = true,
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
dns._debug(true)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [
{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 123456 },
{ name => "l.www.google.com", ipv6 => "::1", ttl => 10 },
],
}
--- request
GET /t
--- error_log
lru_cache SET: www.google.com|1 10
shared_dict SET: www.google.com|1 10
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":123456},{"address":"0:0:0:0:0:0:0:1","type":28,"class":1,"name":"l.www.google.com","ttl":10}]
=== TEST 13: Cache TTLs not minimised by default
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}}
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
dns._debug(true)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [
{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 123456 },
{ name => "l.www.google.com", ipv6 => "::1", ttl => 10 },
],
}
--- request
GET /t
--- error_log
lru_cache SET: www.google.com|1 123456
shared_dict SET: www.google.com|1 123456
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":123456},{"address":"0:0:0:0:0:0:0:1","type":28,"class":1,"name":"l.www.google.com","ttl":10}]

View file

@ -0,0 +1,275 @@
use lib 't';
use TestDNS;
use Cwd qw(cwd);
plan tests => repeat_each() * 17;
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
lua_socket_log_errors off;
};
no_long_string();
run_tests();
__DATA__
=== TEST 1: Query is triggered when cache is expired
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}},
max_stale = 10
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
dns._debug(true)
-- Sleep beyond response TTL
ngx.sleep(1.1)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
if stale then
answer = stale
else
ngx.say(err)
end
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
ngx.sleep(0.1)
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 1 }],
}
--- request
GET /t
--- error_log
Returning STALE
Attempting to repopulate 'www.google.com'
Repopulating 'www.google.com'
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":1}]
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":0}]
=== TEST 2: Query is not triggered when cache expires and max_stale is disabled
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}, retrans = 1, timeout = 50 },
max_stale = 0
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
dns._debug(true)
-- Sleep beyond response TTL
ngx.sleep(1.1)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
if stale then
answer = stale
else
ngx.say(err)
end
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
ngx.sleep(0.1)
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 1 }],
}
--- request
GET /t
--- no_error_log
Attempting to repopulate 'www.google.com'
Repopulating 'www.google.com'
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":0}]
=== TEST 3: Repopulate ignores max_stale
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}, retrans = 1, timeout = 50 },
max_stale = 10,
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
dns._debug(true)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
-- Sleep beyond response TTL
ngx.sleep(1.1)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
if stale then
answer = stale
else
ngx.say(err)
end
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
ngx.sleep(0.1)
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 1 }],
}
--- request
GET /t
--- error_log
Repopulating 'www.google.com'
Querying: www.google.com
Resolver error
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":0}]
=== TEST 4: Multiple queries only trigger 1 repopulate timer
--- http_config eval
"$::HttpConfig"
. q{
lua_shared_dict dns_cache 1m;
init_by_lua '
local DNS_Cache = require("resty.dns.cache")
DNS_Cache.init_cache()
';
}
--- config
location /t {
content_by_lua '
local DNS_Cache = require("resty.dns.cache")
local dns, err = DNS_Cache.new({
dict = "dns_cache",
resolver = {nameservers = {{"127.0.0.1", "1953"}}, retrans = 1, timeout = 50 },
repopulate = true,
})
if not dns then
ngx.say(err)
end
dns.resolver._id = 125
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
dns._debug(true)
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
local answer, err, stale = dns:query("www.google.com", { qtype = dns.TYPE_A })
if not answer then
ngx.say(err)
end
local cjson = require"cjson"
ngx.say(cjson.encode(answer))
';
}
--- udp_listen: 1953
--- udp_reply dns
{
id => 125,
opcode => 0,
qname => 'www.google.com',
answer => [{ name => "www.google.com", ipv4 => "127.0.0.1", ttl => 1 }],
}
--- request
GET /t
--- no_error_log
Attempting to repopulate www.google.com
--- response_body
[{"address":"127.0.0.1","type":1,"class":1,"name":"www.google.com","ttl":1}]

View file

@ -0,0 +1,271 @@
package TestDNS;
use strict;
use warnings;
use 5.010001;
use Test::Nginx::Socket::Lua -Base;
#use JSON::XS;
use constant {
TYPE_A => 1,
TYPE_TXT => 16,
TYPE_CNAME => 5,
TYPE_AAAA => 28,
CLASS_INTERNET => 1,
};
sub encode_name ($);
sub encode_ipv4 ($);
sub encode_ipv6 ($);
sub gen_dns_reply ($$);
sub Test::Base::Filter::dns {
my ($self, $code) = @_;
my $args = $self->current_arguments;
#warn "args: $args";
if (defined $args && $args ne 'tcp' && $args ne 'udp') {
die "Invalid argument to the \"dns\" filter: $args\n";
}
my $mode = $args // 'udp';
my $block = $self->current_block;
my $pointer_spec = $block->dns_pointers;
my @pointers;
if (defined $pointer_spec) {
my @loops = split /\s*,\s*/, $pointer_spec;
for my $loop (@loops) {
my @nodes = split /\s*=>\s*/, $loop;
my $prev;
for my $n (@nodes) {
if ($n !~ /^\d+$/ || $n == 0) {
die "bad name ID in the --- dns_pointers: $n\n";
}
if (!defined $prev) {
$prev = $n;
next;
}
$pointers[$prev] = $n;
}
}
}
my $input = eval $code;
if ($@) {
die "failed to evaluate code $code: $@\n";
}
if (!ref $input) {
return $input;
}
if (ref $input eq 'ARRAY') {
my @replies;
for my $t (@$input) {
push @replies, gen_dns_reply($t, $mode);
}
return \@replies;
}
if (ref $input eq 'HASH') {
return gen_dns_reply($input, $mode);
}
return $input;
}
sub gen_dns_reply ($$) {
my ($t, $mode) = @_;
my @raw_names;
push @raw_names, \($t->{qname});
my $answers = $t->{answer} // [];
if (!ref $answers) {
$answers = [$answers];
}
for my $ans (@$answers) {
push @raw_names, \($ans->{name});
if (defined $ans->{cname}) {
push @raw_names, \($ans->{cname});
}
}
for my $rname (@raw_names) {
$$rname = encode_name($$rname // "");
}
my $qname = $t->{qname};
my $s = '';
my $id = $t->{id} // 0;
$s .= pack("n", $id);
#warn "id: ", length($s), " ", encode_json([$s]);
my $qr = $t->{qr} // 1;
my $opcode = $t->{opcode} // 0;
my $aa = $t->{aa} // 0;
my $tc = $t->{tc} // 0;
my $rd = $t->{rd} // 1;
my $ra = $t->{ra} // 1;
my $rcode = $t->{rcode} // 0;
my $flags = ($qr << 15) + ($opcode << 11) + ($aa << 10) + ($tc << 9) + ($rd << 8) + ($ra << 7) + $rcode;
#warn sprintf("flags: %b", $flags);
$flags = pack("n", $flags);
$s .= $flags;
#warn "flags: ", length($flags), " ", encode_json([$flags]);
my $qdcount = $t->{qdcount} // 1;
my $ancount = $t->{ancount} // scalar @$answers;
my $nscount = 0;
my $arcount = 0;
$s .= pack("nnnn", $qdcount, $ancount, $nscount, $arcount);
#warn "qname: ", length($qname), " ", encode_json([$qname]);
$s .= $qname;
my $qs_type = $t->{qtype} // TYPE_A;
my $qs_class = $t->{qclass} // CLASS_INTERNET;
$s .= pack("nn", $qs_type, $qs_class);
for my $ans (@$answers) {
my $name = $ans->{name};
my $type = $ans->{type};
my $class = $ans->{class};
my $ttl = $ans->{ttl};
my $rdlength = $ans->{rdlength};
my $rddata = $ans->{rddata};
my $ipv4 = $ans->{ipv4};
if (defined $ipv4) {
my ($data, $len) = encode_ipv4($ipv4);
$rddata //= $data;
$rdlength //= $len;
$type //= TYPE_A;
$class //= CLASS_INTERNET;
}
my $ipv6 = $ans->{ipv6};
if (defined $ipv6) {
my ($data, $len) = encode_ipv6($ipv6);
$rddata //= $data;
$rdlength //= $len;
$type //= TYPE_AAAA;
$class //= CLASS_INTERNET;
}
my $cname = $ans->{cname};
if (defined $cname) {
$rddata //= $cname;
$rdlength //= length $rddata;
$type //= TYPE_CNAME;
$class //= CLASS_INTERNET;
}
my $txt = $ans->{txt};
if (defined $txt) {
$rddata //= $txt;
$rdlength //= length $rddata;
$type //= TYPE_TXT;
$class //= CLASS_INTERNET;
}
$type //= 0;
$class //= 0;
$ttl //= 0;
#warn "rdlength: $rdlength, rddata: ", encode_json([$rddata]), "\n";
$s .= $name . pack("nnNn", $type, $class, $ttl, $rdlength) . $rddata;
}
if ($mode eq 'tcp') {
return pack("n", length($s)) . $s;
}
return $s;
}
sub encode_ipv4 ($) {
my $txt = shift;
my @bytes = split /\./, $txt;
return pack("CCCC", @bytes), 4;
}
sub encode_ipv6 ($) {
my $txt = shift;
my @groups = split /:/, $txt;
my $nils = 0;
my $nonnils = 0;
for my $g (@groups) {
if ($g eq '') {
$nils++;
} else {
$nonnils++;
$g = hex($g);
}
}
my $total = $nils + $nonnils;
if ($total > 8 ) {
die "Invalid IPv6 address: too many groups: $total: $txt";
}
if ($nils) {
my $found = 0;
my @new_groups;
for my $g (@groups) {
if ($g eq '') {
if ($found) {
next;
}
for (1 .. 8 - $nonnils) {
push @new_groups, 0;
}
$found = 1;
} else {
push @new_groups, $g;
}
}
@groups = @new_groups;
}
if (@groups != 8) {
die "Invalid IPv6 address: $txt: @groups\n";
}
#warn "IPv6 groups: @groups";
return pack("nnnnnnnn", @groups), 16;
}
sub encode_name ($) {
my $name = shift;
$name =~ s/([^.]+)\.?/chr(length($1)) . $1/ge;
$name .= "\0";
return $name;
}
1

View file

@ -0,0 +1,32 @@
#!/usr/bin/env perl
use strict;
use warnings;
sub file_contains ($$);
my $version;
for my $file (map glob, qw{ *.lua lib/*.lua lib/*/*.lua lib/*/*/*.lua lib/*/*/*/*.lua lib/*/*/*/*/*.lua }) {
print "Checking use of Lua global variables in file $file ...\n";
system("luac -p -l $file | grep ETGLOBAL | grep -vE 'require|type|tostring|error|ngx|ndk|jit|setmetatable|getmetatable|string|table|io|os|print|tonumber|math|pcall|xpcall|unpack|pairs|ipairs|assert|module|package|coroutine|[gs]etfenv|next|rawget|rawset|rawlen'");
file_contains($file, "attempt to write to undeclared variable");
#system("grep -H -n -E --color '.{81}' $file");
}
sub file_contains ($$) {
my ($file, $regex) = @_;
open my $in, $file
or die "Cannot open $file fo reading: $!\n";
my $content = do { local $/; <$in> };
close $in;
#print "$content";
return scalar ($content =~ /$regex/);
}
if (-d 't') {
for my $file (map glob, qw{ t/*.t t/*/*.t t/*/*/*.t }) {
system(qq{grep -H -n --color -E '\\--- ?(ONLY|LAST)' $file});
}
}

View file

@ -0,0 +1 @@
*.t linguist-language=Text

View file

@ -0,0 +1,10 @@
*.swp
*.swo
*~
go
t/servroot/
reindex
nginx
ctags
tags
a.lua

View file

@ -0,0 +1,18 @@
OPENRESTY_PREFIX=/usr/local/openresty
PREFIX ?= /usr/local
LUA_INCLUDE_DIR ?= $(PREFIX)/include
LUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION)
INSTALL ?= install
.PHONY: all test install
all: ;
install: all
$(INSTALL) -d $(DESTDIR)/$(LUA_LIB_DIR)/resty/dns
$(INSTALL) lib/resty/dns/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/resty/dns/
test: all
PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$$PATH prove -I../test-nginx/lib -r t

View file

@ -0,0 +1,404 @@
Name
====
lua-resty-dns - Lua DNS resolver for the ngx_lua based on the cosocket API
Table of Contents
=================
* [Name](#name)
* [Status](#status)
* [Description](#description)
* [Synopsis](#synopsis)
* [Methods](#methods)
* [new](#new)
* [query](#query)
* [tcp_query](#tcp_query)
* [set_timeout](#set_timeout)
* [compress_ipv6_addr](#compress_ipv6_addr)
* [Constants](#constants)
* [TYPE_A](#type_a)
* [TYPE_NS](#type_ns)
* [TYPE_CNAME](#type_cname)
* [TYPE_PTR](#type_ptr)
* [TYPE_MX](#type_mx)
* [TYPE_TXT](#type_txt)
* [TYPE_AAAA](#type_aaaa)
* [TYPE_SRV](#type_srv)
* [TYPE_SPF](#type_spf)
* [CLASS_IN](#class_in)
* [Automatic Error Logging](#automatic-error-logging)
* [Limitations](#limitations)
* [TODO](#todo)
* [Author](#author)
* [Copyright and License](#copyright-and-license)
* [See Also](#see-also)
Status
======
This library is considered production ready.
Description
===========
This Lua library provies a DNS resolver for the ngx_lua nginx module:
http://wiki.nginx.org/HttpLuaModule
This Lua library takes advantage of ngx_lua's cosocket API, which ensures
100% nonblocking behavior.
Note that at least [ngx_lua 0.5.12](https://github.com/chaoslawful/lua-nginx-module/tags) or [ngx_openresty 1.2.1.11](http://openresty.org/#Download) is required.
Also, the [bit library](http://bitop.luajit.org/) is also required. If you're using LuaJIT 2.0 with ngx_lua, then the `bit` library is already available by default.
Note that, this library is bundled and enabled by default in the [ngx_openresty bundle](http://openresty.org/).
Synopsis
========
```lua
lua_package_path "/path/to/lua-resty-dns/lib/?.lua;;";
server {
location = /dns {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{
nameservers = {"8.8.8.8", {"8.8.4.4", 53} },
retrans = 5, -- 5 retransmissions on receive timeout
timeout = 2000, -- 2 sec
}
if not r then
ngx.say("failed to instantiate the resolver: ", err)
return
end
local answers, err = r:query("www.google.com")
if not answers then
ngx.say("failed to query the DNS server: ", err)
return
end
if answers.errcode then
ngx.say("server returned error code: ", answers.errcode,
": ", answers.errstr)
end
for i, ans in ipairs(answers) do
ngx.say(ans.name, " ", ans.address or ans.cname,
" type:", ans.type, " class:", ans.class,
" ttl:", ans.ttl)
end
';
}
}
```
[Back to TOC](#table-of-contents)
Methods
=======
[Back to TOC](#table-of-contents)
new
---
`syntax: r, err = class:new(opts)`
Creates a dns.resolver object. Returns `nil` and an message string on error.
It accepts a `opts` table argument. The following options are supported:
* `nameservers`
a list of nameservers to be used. Each nameserver entry can be either a single hostname string or a table holding both the hostname string and the port number. The nameserver is picked up by a simple round-robin algorithm for each `query` method call. This option is required.
* `retrans`
the total number of times of retransmitting the DNS request when receiving a DNS response times out according to the `timeout` setting. Default to `5` times. When trying to retransmit the query, the next nameserver according to the round-robin algorithm will be picked up.
* `timeout`
the time in milliseconds for waiting for the respond for a single attempt of request transmition. note that this is ''not'' the maximal total waiting time before giving up, the maximal total waiting time can be calculated by the expression `timeout x retrans`. The `timeout` setting can also be changed by calling the `set_timeout` method. The default `timeout` setting is 2000 milliseconds, or 2 seconds.
* `no_recurse`
a boolean flag controls whether to disable the "recursion desired" (RD) flag in the UDP request. Default to `false`.
[Back to TOC](#table-of-contents)
query
-----
`syntax: answers, err = r:query(name, options?)`
Performs a DNS standard query to the nameservers specified by the `new` method,
and returns all the answer records in an array-like Lua table. In case of errors, it will
return `nil` and a string describing the error instead.
If the server returns a non-zero error code, the fields `errcode` and `errstr` will be set accordingly in the Lua table returned.
Each entry in the `answers` returned table value is also a hash-like Lua table
which usually takes some of the following fields:
* `name`
The resource record name.
* `type`
The current resource record type, possible values are `1` (`TYPE_A`), `5` (`TYPE_CNAME`), `28` (`TYPE_AAAA`), and any other values allowed by RFC 1035.
* `address`
The IPv4 or IPv6 address in their textual representations when the resource record type is either `1` (`TYPE_A`) or `28` (`TYPE_AAAA`), respectively. Secussesive 16-bit zero groups in IPv6 addresses will not be compressed by default, if you want that, you need to call the `compress_ipv6_addr` static method instead.
* `cname`
The (decoded) record data value for `CNAME` resource records. Only present for `CNAME` records.
* `ttl`
The time-to-live (TTL) value in seconds for the current resource record.
* `class`
The current resource record class, possible values are `1` (`CLASS_IN`) or any other values allowed by RFC 1035.
* `preference`
The preference integer number for `MX` resource records. Only present for `MX` type records.
* `exchange`
The exchange domain name for `MX` resource records. Only present for `MX` type records.
* `nsdname`
A domain-name which specifies a host which should be authoritative for the specified class and domain. Usually present for `NS` type records.
* `rdata`
The raw resource data (RDATA) for resource records that are not recognized.
* `txt`
The record value for `TXT` records. When there is only one character string in this record, then this field takes a single Lua string. Otherwise this field takes a Lua table holding all the strings.
* `ptrdname`
The record value for `PTR` records.
This method also takes an optional `options` argument table, which takes the following fields:
* `qtype`
The type of the question. Possible values are `1` (`TYPE_A`), `5` (`TYPE_CNAME`), `28` (`TYPE_AAAA`), or any other QTYPE value specified by RFC 1035 and RFC 3596. Default to `1` (`TYPE_A`).
When data truncation happens, the resolver will automatically retry using the TCP transport mode
to query the current nameserver. All TCP connections are short lived.
[Back to TOC](#table-of-contents)
tcp_query
---------
`syntax: answers, err = r:tcp_query(name, options?)`
Just like the `query` method, but enforce the TCP transport mode instead of UDP.
All TCP connections are short lived.
Here is an example:
```lua
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{
nameservers = { "8.8.8.8" }
}
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:tcp_query("www.google.com", { qtype = r.TYPE_A })
if not ans then
ngx.say("failed to query: ", err)
return
end
local cjson = require "cjson"
ngx.say("records: ", cjson.encode(ans))
```
[Back to TOC](#table-of-contents)
set_timeout
-----------
`syntax: r:set_timeout(time)`
Overrides the current `timeout` setting by the `time` argument in milliseconds for all the nameserver peers.
[Back to TOC](#table-of-contents)
compress_ipv6_addr
------------------
`syntax: compressed = resty.dns.resolver.compress_ipv6_addr(address)`
Compresses the successive 16-bit zero groups in the textual format of the IPv6 address.
For example,
```lua
local resolver = require "resty.dns.resolver"
local compress = resolver.compress_ipv6_addr
local new_addr = compress("FF01:0:0:0:0:0:0:101")
```
will yield `FF01::101` in the `new_addr` return value.
[Back to TOC](#table-of-contents)
Constants
=========
[Back to TOC](#table-of-contents)
TYPE_A
------
The `A` resource record type, equal to the decimal number `1`.
[Back to TOC](#table-of-contents)
TYPE_NS
-------
The `NS` resource record type, equal to the decimal number `2`.
[Back to TOC](#table-of-contents)
TYPE_CNAME
----------
The `CNAME` resource record type, equal to the decimal number `5`.
[Back to TOC](#table-of-contents)
TYPE_PTR
--------
The `PTR` resource record type, equal to the decimal number `12`.
[Back to TOC](#table-of-contents)
TYPE_MX
-------
The `MX` resource record type, equal to the decimal number `15`.
[Back to TOC](#table-of-contents)
TYPE_TXT
--------
The `TXT` resource record type, equal to the decimal number `16`.
[Back to TOC](#table-of-contents)
TYPE_AAAA
---------
`syntax: typ = r.TYPE_AAAA`
The `AAAA` resource record type, equal to the decimal number `28`.
[Back to TOC](#table-of-contents)
TYPE_SRV
---------
`syntax: typ = r.TYPE_SRV`
The `SRV` resource record type, equal to the decimal number `33`.
See RFC 2782 for details.
[Back to TOC](#table-of-contents)
TYPE_SPF
---------
`syntax: typ = r.TYPE_SPF`
The `SPF` resource record type, equal to the decimal number `99`.
See RFC 4408 for details.
[Back to TOC](#table-of-contents)
CLASS_IN
--------
`syntax: class = r.CLASS_IN`
The `Internet` resource record type, equal to the decimal number `1`.
[Back to TOC](#table-of-contents)
Automatic Error Logging
=======================
By default the underlying [ngx_lua](http://wiki.nginx.org/HttpLuaModule) module
does error logging when socket errors happen. If you are already doing proper error
handling in your own Lua code, then you are recommended to disable this automatic error logging by turning off [ngx_lua](http://wiki.nginx.org/HttpLuaModule)'s [lua_socket_log_errors](http://wiki.nginx.org/HttpLuaModule#lua_socket_log_errors) directive, that is,
```nginx
lua_socket_log_errors off;
```
[Back to TOC](#table-of-contents)
Limitations
===========
* This library cannot be used in code contexts like set_by_lua*, log_by_lua*, and
header_filter_by_lua* where the ngx_lua cosocket API is not available.
* The `resty.dns.resolver` object instance cannot be stored in a Lua variable at the Lua module level,
because it will then be shared by all the concurrent requests handled by the same nginx
worker process (see
http://wiki.nginx.org/HttpLuaModule#Data_Sharing_within_an_Nginx_Worker ) and
result in bad race conditions when concurrent requests are trying to use the same `resty.dns.resolver` instance.
You should always initiate `resty.dns.resolver` objects in function local
variables or in the `ngx.ctx` table. These places all have their own data copies for
each request.
[Back to TOC](#table-of-contents)
TODO
====
* Concurrent (or parallel) query mode
* Better support for other resource record types like `TLSA`.
[Back to TOC](#table-of-contents)
Author
======
Yichun "agentzh" Zhang (章亦春) <agentzh@gmail.com>, CloudFlare Inc.
[Back to TOC](#table-of-contents)
Copyright and License
=====================
This module is licensed under the BSD license.
Copyright (C) 2012-2014, by Yichun "agentzh" Zhang (章亦春) <agentzh@gmail.com>, CloudFlare Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
[Back to TOC](#table-of-contents)
See Also
========
* the ngx_lua module: http://wiki.nginx.org/HttpLuaModule
* the [lua-resty-memcached](https://github.com/agentzh/lua-resty-memcached) library.
* the [lua-resty-redis](https://github.com/agentzh/lua-resty-redis) library.
* the [lua-resty-mysql](https://github.com/agentzh/lua-resty-mysql) library.
[Back to TOC](#table-of-contents)

View file

@ -0,0 +1,803 @@
-- Copyright (C) Yichun Zhang (agentzh)
-- local socket = require "socket"
local bit = require "bit"
local udp = ngx.socket.udp
local rand = math.random
local char = string.char
local byte = string.byte
local find = string.find
local gsub = string.gsub
local sub = string.sub
local format = string.format
local band = bit.band
local rshift = bit.rshift
local lshift = bit.lshift
local insert = table.insert
local concat = table.concat
local re_sub = ngx.re.sub
local tcp = ngx.socket.tcp
local log = ngx.log
local DEBUG = ngx.DEBUG
local randomseed = math.randomseed
local ngx_time = ngx.time
local setmetatable = setmetatable
local type = type
local DOT_CHAR = byte(".")
local TYPE_A = 1
local TYPE_NS = 2
local TYPE_CNAME = 5
local TYPE_PTR = 12
local TYPE_MX = 15
local TYPE_TXT = 16
local TYPE_AAAA = 28
local TYPE_SRV = 33
local TYPE_SPF = 99
local CLASS_IN = 1
local _M = {
_VERSION = '0.14',
TYPE_A = TYPE_A,
TYPE_NS = TYPE_NS,
TYPE_CNAME = TYPE_CNAME,
TYPE_PTR = TYPE_PTR,
TYPE_MX = TYPE_MX,
TYPE_TXT = TYPE_TXT,
TYPE_AAAA = TYPE_AAAA,
TYPE_SRV = TYPE_SRV,
TYPE_SPF = TYPE_SPF,
CLASS_IN = CLASS_IN,
}
local resolver_errstrs = {
"format error", -- 1
"server failure", -- 2
"name error", -- 3
"not implemented", -- 4
"refused", -- 5
}
local mt = { __index = _M }
function _M.new(class, opts)
if not opts then
return nil, "no options table specified"
end
local servers = opts.nameservers
if not servers or #servers == 0 then
return nil, "no nameservers specified"
end
local timeout = opts.timeout or 2000 -- default 2 sec
local n = #servers
local socks = {}
for i = 1, n do
local server = servers[i]
local sock, err = udp()
if not sock then
return nil, "failed to create udp socket: " .. err
end
local host, port
if type(server) == 'table' then
host = server[1]
port = server[2] or 53
else
host = server
port = 53
servers[i] = {host, port}
end
local ok, err = sock:setpeername(host, port)
if not ok then
return nil, "failed to set peer name: " .. err
end
sock:settimeout(timeout)
insert(socks, sock)
end
local tcp_sock, err = tcp()
if not tcp_sock then
return nil, "failed to create tcp socket: " .. err
end
tcp_sock:settimeout(timeout)
return setmetatable(
{ cur = rand(1, n), socks = socks,
tcp_sock = tcp_sock,
servers = servers,
retrans = opts.retrans or 5,
no_recurse = opts.no_recurse,
}, mt)
end
local function pick_sock(self, socks)
local cur = self.cur
if cur == #socks then
self.cur = 1
else
self.cur = cur + 1
end
return socks[cur]
end
local function _get_cur_server(self)
local cur = self.cur
local servers = self.servers
if cur == 1 then
return servers[#servers]
end
return servers[cur - 1]
end
function _M.set_timeout(self, timeout)
local socks = self.socks
if not socks then
return nil, "not initialized"
end
for i = 1, #socks do
local sock = socks[i]
sock:settimeout(timeout)
end
local tcp_sock = self.tcp_sock
if not tcp_sock then
return nil, "not initialized"
end
tcp_sock:settimeout(timeout)
end
local function _encode_name(s)
return char(#s) .. s
end
local function _decode_name(buf, pos)
local labels = {}
local nptrs = 0
local p = pos
while nptrs < 128 do
local fst = byte(buf, p)
if not fst then
return nil, 'truncated';
end
-- print("fst at ", p, ": ", fst)
if fst == 0 then
if nptrs == 0 then
pos = pos + 1
end
break
end
if band(fst, 0xc0) ~= 0 then
-- being a pointer
if nptrs == 0 then
pos = pos + 2
end
nptrs = nptrs + 1
local snd = byte(buf, p + 1)
if not snd then
return nil, 'truncated'
end
p = lshift(band(fst, 0x3f), 8) + snd + 1
-- print("resolving ptr ", p, ": ", byte(buf, p))
else
-- being a label
local label = sub(buf, p + 1, p + fst)
insert(labels, label)
-- print("resolved label ", label)
p = p + fst + 1
if nptrs == 0 then
pos = p
end
end
end
return concat(labels, "."), pos
end
local function _build_request(qname, id, no_recurse, opts)
local qtype
if opts then
qtype = opts.qtype
end
if not qtype then
qtype = 1 -- A record
end
local ident_hi = char(rshift(id, 8))
local ident_lo = char(band(id, 0xff))
local flags
if no_recurse then
-- print("found no recurse")
flags = "\0\0"
else
flags = "\1\0"
end
local nqs = "\0\1"
local nan = "\0\0"
local nns = "\0\0"
local nar = "\0\0"
local typ = "\0" .. char(qtype)
local class = "\0\1" -- the Internet class
if byte(qname, 1) == DOT_CHAR then
return nil, "bad name"
end
local name = gsub(qname, "([^.]+)%.?", _encode_name) .. '\0'
return {
ident_hi, ident_lo, flags, nqs, nan, nns, nar,
name, typ, class
}
end
local function parse_response(buf, id)
local n = #buf
if n < 12 then
return nil, 'truncated';
end
-- header layout: ident flags nqs nan nns nar
local ident_hi = byte(buf, 1)
local ident_lo = byte(buf, 2)
local ans_id = lshift(ident_hi, 8) + ident_lo
-- print("id: ", id, ", ans id: ", ans_id)
if ans_id ~= id then
-- identifier mismatch and throw it away
log(DEBUG, "id mismatch in the DNS reply: ", ans_id, " ~= ", id)
return nil, "id mismatch"
end
local flags_hi = byte(buf, 3)
local flags_lo = byte(buf, 4)
local flags = lshift(flags_hi, 8) + flags_lo
-- print(format("flags: 0x%x", flags))
if band(flags, 0x8000) == 0 then
return nil, format("bad QR flag in the DNS response")
end
if band(flags, 0x200) ~= 0 then
return nil, "truncated"
end
local code = band(flags, 0x7f)
-- print(format("code: %d", code))
local nqs_hi = byte(buf, 5)
local nqs_lo = byte(buf, 6)
local nqs = lshift(nqs_hi, 8) + nqs_lo
-- print("nqs: ", nqs)
if nqs ~= 1 then
return nil, format("bad number of questions in DNS response: %d", nqs)
end
local nan_hi = byte(buf, 7)
local nan_lo = byte(buf, 8)
local nan = lshift(nan_hi, 8) + nan_lo
-- print("nan: ", nan)
-- skip the question part
local ans_qname, pos = _decode_name(buf, 13)
if not ans_qname then
return nil, pos
end
-- print("qname in reply: ", ans_qname)
-- print("question: ", sub(buf, 13, pos))
if pos + 3 + nan * 12 > n then
-- print(format("%d > %d", pos + 3 + nan * 12, n))
return nil, 'truncated';
end
-- question section layout: qname qtype(2) qclass(2)
local type_hi = byte(buf, pos)
local type_lo = byte(buf, pos + 1)
local ans_type = lshift(type_hi, 8) + type_lo
-- print("ans qtype: ", ans_type)
local class_hi = byte(buf, pos + 2)
local class_lo = byte(buf, pos + 3)
local qclass = lshift(class_hi, 8) + class_lo
-- print("ans qclass: ", qclass)
if qclass ~= 1 then
return nil, format("unknown query class %d in DNS response", qclass)
end
pos = pos + 4
local answers = {}
if code ~= 0 then
answers.errcode = code
answers.errstr = resolver_errstrs[code] or "unknown"
end
for i = 1, nan do
-- print(format("ans %d: qtype:%d qclass:%d", i, qtype, qclass))
local ans = {}
insert(answers, ans)
local name
name, pos = _decode_name(buf, pos)
if not name then
return nil, pos
end
ans.name = name
-- print("name: ", name)
type_hi = byte(buf, pos)
type_lo = byte(buf, pos + 1)
local typ = lshift(type_hi, 8) + type_lo
ans.type = typ
-- print("type: ", typ)
class_hi = byte(buf, pos + 2)
class_lo = byte(buf, pos + 3)
local class = lshift(class_hi, 8) + class_lo
ans.class = class
-- print("class: ", class)
local ttl_bytes = { byte(buf, pos + 4, pos + 7) }
-- print("ttl bytes: ", concat(ttl_bytes, " "))
local ttl = lshift(ttl_bytes[1], 24) + lshift(ttl_bytes[2], 16)
+ lshift(ttl_bytes[3], 8) + ttl_bytes[4]
-- print("ttl: ", ttl)
ans.ttl = ttl
local len_hi = byte(buf, pos + 8)
local len_lo = byte(buf, pos + 9)
local len = lshift(len_hi, 8) + len_lo
-- print("record len: ", len)
pos = pos + 10
if typ == TYPE_A then
if len ~= 4 then
return nil, "bad A record value length: " .. len
end
local addr_bytes = { byte(buf, pos, pos + 3) }
local addr = concat(addr_bytes, ".")
-- print("ipv4 address: ", addr)
ans.address = addr
pos = pos + 4
elseif typ == TYPE_CNAME then
local cname, p = _decode_name(buf, pos)
if not cname then
return nil, pos
end
if p - pos ~= len then
return nil, format("bad cname record length: %d ~= %d",
p - pos, len)
end
pos = p
-- print("cname: ", cname)
ans.cname = cname
elseif typ == TYPE_AAAA then
if len ~= 16 then
return nil, "bad AAAA record value length: " .. len
end
local addr_bytes = { byte(buf, pos, pos + 15) }
local flds = {}
local comp_begin, comp_end
for i = 1, 16, 2 do
local a = addr_bytes[i]
local b = addr_bytes[i + 1]
if a == 0 then
insert(flds, format("%x", b))
else
insert(flds, format("%x%02x", a, b))
end
end
-- we do not compress the IPv6 addresses by default
-- due to performance considerations
ans.address = concat(flds, ":")
pos = pos + 16
elseif typ == TYPE_MX then
-- print("len = ", len)
if len < 3 then
return nil, "bad MX record value length: " .. len
end
local pref_hi = byte(buf, pos)
local pref_lo = byte(buf, pos + 1)
ans.preference = lshift(pref_hi, 8) + pref_lo
local host, p = _decode_name(buf, pos + 2)
if not host then
return nil, pos
end
if p - pos ~= len then
return nil, format("bad cname record length: %d ~= %d",
p - pos, len)
end
ans.exchange = host
pos = p
elseif typ == TYPE_SRV then
if len < 7 then
return nil, "bad SRV record value length: " .. len
end
local prio_hi = byte(buf, pos)
local prio_lo = byte(buf, pos + 1)
ans.priority = lshift(prio_hi, 8) + prio_lo
local weight_hi = byte(buf, pos + 2)
local weight_lo = byte(buf, pos + 3)
ans.weight = lshift(weight_hi, 8) + weight_lo
local port_hi = byte(buf, pos + 4)
local port_lo = byte(buf, pos + 5)
ans.port = lshift(port_hi, 8) + port_lo
local name, p = _decode_name(buf, pos + 6)
if not name then
return nil, pos
end
if p - pos ~= len then
return nil, format("bad srv record length: %d ~= %d",
p - pos, len)
end
ans.target = name
pos = p
elseif typ == TYPE_NS then
local name, p = _decode_name(buf, pos)
if not name then
return nil, pos
end
if p - pos ~= len then
return nil, format("bad cname record length: %d ~= %d",
p - pos, len)
end
pos = p
-- print("name: ", name)
ans.nsdname = name
elseif typ == TYPE_TXT or typ == TYPE_SPF then
local key = (typ == TYPE_TXT) and "txt" or "spf"
local slen = byte(buf, pos)
if slen + 1 > len then
-- truncate the over-run TXT record data
slen = len
end
-- print("slen: ", len)
local val = sub(buf, pos + 1, pos + slen)
local last = pos + len
pos = pos + slen + 1
if pos < last then
-- more strings to be processed
-- this code path is usually cold, so we do not
-- merge the following loop on this code path
-- with the processing logic above.
val = {val}
local idx = 2
repeat
local slen = byte(buf, pos)
if pos + slen + 1 > last then
-- truncate the over-run TXT record data
slen = last - pos - 1
end
val[idx] = sub(buf, pos + 1, pos + slen)
idx = idx + 1
pos = pos + slen + 1
until pos >= last
end
ans[key] = val
elseif typ == TYPE_PTR then
local name, p = _decode_name(buf, pos)
if not name then
return nil, pos
end
if p - pos ~= len then
return nil, format("bad cname record length: %d ~= %d",
p - pos, len)
end
pos = p
-- print("name: ", name)
ans.ptrdname = name
else
-- for unknown types, just forward the raw value
ans.rdata = sub(buf, pos, pos + len - 1)
pos = pos + len
end
end
return answers
end
local function _gen_id(self)
local id = self._id -- for regression testing
if id then
return id
end
return rand(0, 65535) -- two bytes
end
local function _tcp_query(self, query, id)
local sock = self.tcp_sock
if not sock then
return nil, "not initialized"
end
log(DEBUG, "query the TCP server due to reply truncation")
local server = _get_cur_server(self)
local ok, err = sock:connect(server[1], server[2])
if not ok then
return nil, "failed to connect to TCP server "
.. concat(server, ":") .. ": " .. err
end
query = concat(query, "")
local len = #query
local len_hi = char(rshift(len, 8))
local len_lo = char(band(len, 0xff))
local bytes, err = sock:send({len_hi, len_lo, query})
if not bytes then
return nil, "failed to send query to TCP server "
.. concat(server, ":") .. ": " .. err
end
local buf, err = sock:receive(2)
if not buf then
return nil, "failed to receive the reply length field from TCP server "
.. concat(server, ":") .. ": " .. err
end
local len_hi = byte(buf, 1)
local len_lo = byte(buf, 2)
local len = lshift(len_hi, 8) + len_lo
-- print("tcp message len: ", len)
buf, err = sock:receive(len)
if not buf then
return nil, "failed to receive the reply message body from TCP server "
.. concat(server, ":") .. ": " .. err
end
local answers, err = parse_response(buf, id)
if not answers then
return nil, "failed to parse the reply from the TCP server "
.. concat(server, ":") .. ": " .. err
end
sock:close()
return answers
end
function _M.tcp_query(self, qname, opts)
local socks = self.socks
if not socks then
return nil, "not initialized"
end
pick_sock(self, socks)
local id = _gen_id(self)
local query, err = _build_request(qname, id, self.no_recurse, opts)
if not query then
return nil, err
end
return _tcp_query(self, query, id)
end
function _M.query(self, qname, opts)
local socks = self.socks
if not socks then
return nil, "not initialized"
end
local id = _gen_id(self)
local query, err = _build_request(qname, id, self.no_recurse, opts)
if not query then
return nil, err
end
-- local cjson = require "cjson"
-- print("query: ", cjson.encode(concat(query, "")))
local retrans = self.retrans
-- print("retrans: ", retrans)
for i = 1, retrans do
local sock = pick_sock(self, socks)
local ok, err = sock:send(query)
if not ok then
local server = _get_cur_server(self)
return nil, "failed to send request to UDP server "
.. concat(server, ":") .. ": " .. err
end
local buf, err
for j = 1, 128 do
buf, err = sock:receive(4096)
if err then
break
end
if buf then
local answers
answers, err = parse_response(buf, id)
if not answers then
if err == "truncated" then
return _tcp_query(self, query, id)
end
if err ~= "id mismatch" then
return nil, err
end
-- retry receiving when err == "id mismatch"
else
return answers
end
end
end
if err ~= "timeout" or i == retrans then
local server = _get_cur_server(self)
return nil, "failed to receive reply from UDP server "
.. concat(server, ":") .. ": " .. err
end
end
-- impossible to reach here
end
function _M.compress_ipv6_addr(addr)
local addr = re_sub(addr, "^(0:)+|(:0)+$|:(0:)+", "::", "jo")
if addr == "::0" then
addr = "::"
end
return addr
end
randomseed(ngx_time())
return _M

View file

@ -0,0 +1,271 @@
package TestDNS;
use strict;
use warnings;
use 5.010001;
use Test::Nginx::Socket::Lua -Base;
#use JSON::XS;
use constant {
TYPE_A => 1,
TYPE_TXT => 16,
TYPE_CNAME => 5,
TYPE_AAAA => 28,
CLASS_INTERNET => 1,
};
sub encode_name ($);
sub encode_ipv4 ($);
sub encode_ipv6 ($);
sub gen_dns_reply ($$);
sub Test::Base::Filter::dns {
my ($self, $code) = @_;
my $args = $self->current_arguments;
#warn "args: $args";
if (defined $args && $args ne 'tcp' && $args ne 'udp') {
die "Invalid argument to the \"dns\" filter: $args\n";
}
my $mode = $args // 'udp';
my $block = $self->current_block;
my $pointer_spec = $block->dns_pointers;
my @pointers;
if (defined $pointer_spec) {
my @loops = split /\s*,\s*/, $pointer_spec;
for my $loop (@loops) {
my @nodes = split /\s*=>\s*/, $loop;
my $prev;
for my $n (@nodes) {
if ($n !~ /^\d+$/ || $n == 0) {
die "bad name ID in the --- dns_pointers: $n\n";
}
if (!defined $prev) {
$prev = $n;
next;
}
$pointers[$prev] = $n;
}
}
}
my $input = eval $code;
if ($@) {
die "failed to evaluate code $code: $@\n";
}
if (!ref $input) {
return $input;
}
if (ref $input eq 'ARRAY') {
my @replies;
for my $t (@$input) {
push @replies, gen_dns_reply($t, $mode);
}
return \@replies;
}
if (ref $input eq 'HASH') {
return gen_dns_reply($input, $mode);
}
return $input;
}
sub gen_dns_reply ($$) {
my ($t, $mode) = @_;
my @raw_names;
push @raw_names, \($t->{qname});
my $answers = $t->{answer} // [];
if (!ref $answers) {
$answers = [$answers];
}
for my $ans (@$answers) {
push @raw_names, \($ans->{name});
if (defined $ans->{cname}) {
push @raw_names, \($ans->{cname});
}
}
for my $rname (@raw_names) {
$$rname = encode_name($$rname // "");
}
my $qname = $t->{qname};
my $s = '';
my $id = $t->{id} // 0;
$s .= pack("n", $id);
#warn "id: ", length($s), " ", encode_json([$s]);
my $qr = $t->{qr} // 1;
my $opcode = $t->{opcode} // 0;
my $aa = $t->{aa} // 0;
my $tc = $t->{tc} // 0;
my $rd = $t->{rd} // 1;
my $ra = $t->{ra} // 1;
my $rcode = $t->{rcode} // 0;
my $flags = ($qr << 15) + ($opcode << 11) + ($aa << 10) + ($tc << 9) + ($rd << 8) + ($ra << 7) + $rcode;
#warn sprintf("flags: %b", $flags);
$flags = pack("n", $flags);
$s .= $flags;
#warn "flags: ", length($flags), " ", encode_json([$flags]);
my $qdcount = $t->{qdcount} // 1;
my $ancount = $t->{ancount} // scalar @$answers;
my $nscount = 0;
my $arcount = 0;
$s .= pack("nnnn", $qdcount, $ancount, $nscount, $arcount);
#warn "qname: ", length($qname), " ", encode_json([$qname]);
$s .= $qname;
my $qs_type = $t->{qtype} // TYPE_A;
my $qs_class = $t->{qclass} // CLASS_INTERNET;
$s .= pack("nn", $qs_type, $qs_class);
for my $ans (@$answers) {
my $name = $ans->{name};
my $type = $ans->{type};
my $class = $ans->{class};
my $ttl = $ans->{ttl};
my $rdlength = $ans->{rdlength};
my $rddata = $ans->{rddata};
my $ipv4 = $ans->{ipv4};
if (defined $ipv4) {
my ($data, $len) = encode_ipv4($ipv4);
$rddata //= $data;
$rdlength //= $len;
$type //= TYPE_A;
$class //= CLASS_INTERNET;
}
my $ipv6 = $ans->{ipv6};
if (defined $ipv6) {
my ($data, $len) = encode_ipv6($ipv6);
$rddata //= $data;
$rdlength //= $len;
$type //= TYPE_AAAA;
$class //= CLASS_INTERNET;
}
my $cname = $ans->{cname};
if (defined $cname) {
$rddata //= $cname;
$rdlength //= length $rddata;
$type //= TYPE_CNAME;
$class //= CLASS_INTERNET;
}
my $txt = $ans->{txt};
if (defined $txt) {
$rddata //= $txt;
$rdlength //= length $rddata;
$type //= TYPE_TXT;
$class //= CLASS_INTERNET;
}
$type //= 0;
$class //= 0;
$ttl //= 0;
#warn "rdlength: $rdlength, rddata: ", encode_json([$rddata]), "\n";
$s .= $name . pack("nnNn", $type, $class, $ttl, $rdlength) . $rddata;
}
if ($mode eq 'tcp') {
return pack("n", length($s)) . $s;
}
return $s;
}
sub encode_ipv4 ($) {
my $txt = shift;
my @bytes = split /\./, $txt;
return pack("CCCC", @bytes), 4;
}
sub encode_ipv6 ($) {
my $txt = shift;
my @groups = split /:/, $txt;
my $nils = 0;
my $nonnils = 0;
for my $g (@groups) {
if ($g eq '') {
$nils++;
} else {
$nonnils++;
$g = hex($g);
}
}
my $total = $nils + $nonnils;
if ($total > 8 ) {
die "Invalid IPv6 address: too many groups: $total: $txt";
}
if ($nils) {
my $found = 0;
my @new_groups;
for my $g (@groups) {
if ($g eq '') {
if ($found) {
next;
}
for (1 .. 8 - $nonnils) {
push @new_groups, 0;
}
$found = 1;
} else {
push @new_groups, $g;
}
}
@groups = @new_groups;
}
if (@groups != 8) {
die "Invalid IPv6 address: $txt: @groups\n";
}
#warn "IPv6 groups: @groups";
return pack("nnnnnnnn", @groups), 16;
}
sub encode_name ($) {
my $name = shift;
$name =~ s/([^.]+)\.?/chr(length($1)) . $1/ge;
$name .= "\0";
return $name;
}
1

View file

@ -0,0 +1,89 @@
local ngx_null = ngx.null
local tostring = tostring
local byte = string.byte
local gsub = string.gsub
local sort = table.sort
local pairs = pairs
local ipairs = ipairs
local concat = table.concat
local ok, new_tab = pcall(require, "table.new")
if not ok then
new_tab = function (narr, nrec) return {} end
end
local _M = {}
local metachars = {
['\t'] = '\\t',
["\\"] = "\\\\",
['"'] = '\\"',
['\r'] = '\\r',
['\n'] = '\\n',
}
local function encode_str(s)
-- XXX we will rewrite this when string.buffer is implemented
-- in LuaJIT 2.1 because string.gsub cannot be JIT compiled.
return gsub(s, '["\\\r\n\t]', metachars)
end
local function is_arr(t)
local exp = 1
for k, _ in pairs(t) do
if k ~= exp then
return nil
end
exp = exp + 1
end
return exp - 1
end
local encode
encode = function (v)
if v == nil or v == ngx_null then
return "null"
end
local typ = type(v)
if typ == 'string' then
return '"' .. encode_str(v) .. '"'
end
if typ == 'number' or typ == 'boolean' then
return tostring(v)
end
if typ == 'table' then
local n = is_arr(v)
if n then
local bits = new_tab(n, 0)
for i, elem in ipairs(v) do
bits[i] = encode(elem)
end
return "[" .. concat(bits, ",") .. "]"
end
local keys = {}
local i = 0
for key, _ in pairs(v) do
i = i + 1
keys[i] = key
end
sort(keys)
local bits = new_tab(0, i)
i = 0
for _, key in ipairs(keys) do
i = i + 1
bits[i] = encode(key) .. ":" .. encode(v[key])
end
return "{" .. concat(bits, ",") .. "}"
end
return '"<' .. typ .. '>"'
end
_M.encode = encode
return _M

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,502 @@
# vim:set ft= ts=4 sw=4 et:
use Test::Nginx::Socket::Lua;
use Cwd qw(cwd);
repeat_each(2);
plan tests => repeat_each() * (3 * blocks());
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/t/lib/?.lua;$pwd/lib/?.lua;;";
lua_package_cpath "/usr/local/openresty-debug/lualib/?.so;/usr/local/openresty/lualib/?.so;;";
};
$ENV{TEST_NGINX_RESOLVER} ||= '8.8.8.8';
no_long_string();
run_tests();
__DATA__
=== TEST 1: A records
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{ nameservers = { "$TEST_NGINX_RESOLVER" } }
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:query("www.google.com", { qtype = r.TYPE_A })
if not ans then
ngx.say("failed to query: ", err)
return
end
local cjson = require "cjson"
ngx.say("records: ", cjson.encode(ans))
';
}
--- request
GET /t
--- response_body_like chop
^records: \[.*?"address":"(?:\d{1,3}\.){3}\d+".*?\]$
--- no_error_log
[error]
=== TEST 2: CNAME records
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{ nameservers = { "$TEST_NGINX_RESOLVER" } }
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:query("www.yahoo.com", { qtype = r.TYPE_CNAME })
if not ans then
ngx.say("failed to query: ", err)
return
end
local cjson = require "cjson"
ngx.say("records: ", cjson.encode(ans))
';
}
--- request
GET /t
--- response_body_like chop
^records: \[.*?"cname":"[-_a-z0-9.]+".*?\]$
--- no_error_log
[error]
=== TEST 3: AAAA records
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{ nameservers = { "$TEST_NGINX_RESOLVER" } }
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:query("www.google.com", { qtype = r.TYPE_AAAA })
if not ans then
ngx.say("failed to query: ", err)
return
end
local cjson = require "cjson"
ngx.say("records: ", cjson.encode(ans))
';
}
--- request
GET /t
--- response_body_like chop
^records: \[.*?"address":"[a-fA-F0-9]*(?::[a-fA-F0-9]*)+".*?\]$
--- no_error_log
[error]
=== TEST 4: compress ipv6 addr
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local c = resolver.compress_ipv6_addr
ngx.say(c("1080:0:0:0:8:800:200C:417A"))
ngx.say(c("FF01:0:0:0:0:0:0:101"))
ngx.say(c("0:0:0:0:0:0:0:1"))
ngx.say(c("1:5:0:0:0:0:0:0"))
ngx.say(c("7:25:0:0:0:3:0:0"))
ngx.say(c("0:0:0:0:0:0:0:0"))
';
}
--- request
GET /t
--- response_body
1080::8:800:200C:417A
FF01::101
::1
1:5::
7:25::3:0:0
::
--- no_error_log
[error]
=== TEST 5: A records (TCP)
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{ nameservers = { "$TEST_NGINX_RESOLVER" } }
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:tcp_query("www.google.com", { qtype = r.TYPE_A })
if not ans then
ngx.say("failed to query: ", err)
return
end
local cjson = require "cjson"
ngx.say("records: ", cjson.encode(ans))
';
}
--- request
GET /t
--- response_body_like chop
^records: \[.*?"address":"(?:\d{1,3}\.){3}\d+".*?\]$
--- no_error_log
[error]
=== TEST 6: MX records
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{ nameservers = { "$TEST_NGINX_RESOLVER" } }
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:query("gmail.com", { qtype = r.TYPE_MX })
if not ans then
ngx.say("failed to query: ", err)
return
end
local cjson = require "cjson"
ngx.say("records: ", cjson.encode(ans))
';
}
--- request
GET /t
--- response_body_like chop
^records: \[\{.*?"preference":\d+,.*?"exchange":"[^"]+".*?\}\]$
--- no_error_log
[error]
=== TEST 7: NS records
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{ nameservers = { "$TEST_NGINX_RESOLVER" } }
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:query("agentzh.org", { qtype = r.TYPE_NS })
if not ans then
ngx.say("failed to query: ", err)
return
end
local cjson = require "cjson"
ngx.say("records: ", cjson.encode(ans))
';
}
--- request
GET /t
--- response_body_like chop
^records: \[\{.*?"nsdname":"[^"]+".*?\}\]$
--- no_error_log
[error]
=== TEST 8: TXT query (no ans)
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{ nameservers = { "$TEST_NGINX_RESOLVER" } }
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:query("agentzh.org", { qtype = r.TYPE_TXT })
if not ans then
ngx.say("failed to query: ", err)
return
end
local cjson = require "cjson"
ngx.say("records: ", cjson.encode(ans))
';
}
--- request
GET /t
--- response_body
records: {}
--- no_error_log
[error]
--- timeout: 10
=== TEST 9: TXT query (with ans)
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{ nameservers = { "$TEST_NGINX_RESOLVER" } }
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:query("gmail.com", { qtype = r.TYPE_TXT })
if not ans then
ngx.say("failed to query: ", err)
return
end
local cjson = require "cjson"
ngx.say("records: ", cjson.encode(ans))
';
}
--- request
GET /t
--- response_body_like chop
^records: \[\{.*?"txt":"v=spf\d+\s[^"]+".*?\}\]$
--- no_error_log
[error]
=== TEST 10: PTR query
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{ nameservers = { "$TEST_NGINX_RESOLVER" } }
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:query("4.4.8.8.in-addr.arpa", { qtype = r.TYPE_PTR })
if not ans then
ngx.say("failed to query: ", err)
return
end
local cjson = require "cjson"
ngx.say("records: ", cjson.encode(ans))
';
}
--- request
GET /t
--- response_body_like chop
^records: \[\{.*?"ptrdname":"google-public-dns-b\.google\.com".*?\}\]$
--- no_error_log
[error]
=== TEST 11: domains with a trailing dot
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{ nameservers = { "$TEST_NGINX_RESOLVER" } }
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:query("www.google.com.", { qtype = r.TYPE_A })
if not ans then
ngx.say("failed to query: ", err)
return
end
local cjson = require "cjson"
ngx.say("records: ", cjson.encode(ans))
';
}
--- request
GET /t
--- response_body_like chop
^records: \[.*?"address":"(?:\d{1,3}\.){3}\d+".*?\]$
--- no_error_log
[error]
=== TEST 12: domains with a leading dot
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{ nameservers = { "$TEST_NGINX_RESOLVER" } }
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:query(".www.google.com", { qtype = r.TYPE_A })
if not ans then
ngx.say("failed to query: ", err)
return
end
local cjson = require "cjson"
ngx.say("records: ", cjson.encode(ans))
';
}
--- request
GET /t
--- response_body
failed to query: bad name
--- no_error_log
[error]
=== TEST 13: SRV records or XMPP
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{ nameservers = { "$TEST_NGINX_RESOLVER" } }
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:query("_xmpp-client._tcp.jabber.org", { qtype = r.TYPE_SRV })
if not ans then
ngx.say("failed to query: ", err)
return
end
local ljson = require "ljson"
ngx.say("records: ", ljson.encode(ans))
';
}
--- request
GET /t
--- response_body_like chop
^records: \[(?:{"class":1,"name":"_xmpp-client._tcp.jabber.org","port":\d+,"priority":\d+,"target":"[\w.]+\.jabber.org","ttl":\d+,"type":33,"weight":\d+},?)+\]$
--- no_error_log
[error]
=== TEST 14: SPF query (with ans)
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{ nameservers = { "$TEST_NGINX_RESOLVER" } }
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:query("linkedin.com", { qtype = r.TYPE_SPF })
if not ans then
ngx.say("failed to query: ", err)
return
end
local cjson = require "cjson"
ngx.say("records: ", cjson.encode(ans))
';
}
--- request
GET /t
--- response_body_like chop
^records: \[\{.*?"spf":"v=spf\d+\s[^"]+".*?\}\]$
--- no_error_log
[error]
=== TEST 15: SPF query (no ans)
--- http_config eval: $::HttpConfig
--- config
location /t {
content_by_lua '
local resolver = require "resty.dns.resolver"
local r, err = resolver:new{ nameservers = { "$TEST_NGINX_RESOLVER" } }
if not r then
ngx.say("failed to instantiate resolver: ", err)
return
end
local ans, err = r:query("agentzh.org", { qtype = r.TYPE_SPF })
if not ans then
ngx.say("failed to query: ", err)
return
end
local cjson = require "cjson"
ngx.say("records: ", cjson.encode(ans))
';
}
--- request
GET /t
--- response_body
records: {}
--- no_error_log
[error]
--- timeout: 10

View file

@ -0,0 +1,549 @@
{
<insert_a_suppression_name_here>
Memcheck:Cond
fun:lj_str_new
}
{
<insert_a_suppression_name_here>
Memcheck:Param
write(buf)
fun:__write_nocancel
fun:ngx_log_error_core
fun:ngx_resolver_read_response
}
{
<insert_a_suppression_name_here>
Memcheck:Cond
fun:ngx_sprintf_num
fun:ngx_vslprintf
fun:ngx_log_error_core
fun:ngx_resolver_read_response
fun:ngx_epoll_process_events
fun:ngx_process_events_and_timers
fun:ngx_single_process_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Addr1
fun:ngx_vslprintf
fun:ngx_snprintf
fun:ngx_sock_ntop
fun:ngx_event_accept
}
{
<insert_a_suppression_name_here>
Memcheck:Param
write(buf)
fun:__write_nocancel
fun:ngx_log_error_core
fun:ngx_resolver_read_response
fun:ngx_event_process_posted
fun:ngx_process_events_and_timers
fun:ngx_single_process_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Cond
fun:ngx_sprintf_num
fun:ngx_vslprintf
fun:ngx_log_error_core
fun:ngx_resolver_read_response
fun:ngx_event_process_posted
fun:ngx_process_events_and_timers
fun:ngx_single_process_cycle
fun:main
}
{
<insert_a_suppression_name_here>
exp-sgcheck:SorG
fun:lj_str_new
fun:lua_pushlstring
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
obj:*
}
{
<insert_a_suppression_name_here>
exp-sgcheck:SorG
fun:lj_str_new
fun:lua_pushlstring
}
{
<insert_a_suppression_name_here>
exp-sgcheck:SorG
fun:ngx_http_lua_ndk_set_var_get
}
{
<insert_a_suppression_name_here>
exp-sgcheck:SorG
fun:lj_str_new
fun:lua_getfield
}
{
<insert_a_suppression_name_here>
exp-sgcheck:SorG
fun:lj_str_new
fun:lua_setfield
}
{
<insert_a_suppression_name_here>
exp-sgcheck:SorG
fun:ngx_http_variables_init_vars
fun:ngx_http_block
}
{
<insert_a_suppression_name_here>
exp-sgcheck:SorG
fun:ngx_conf_parse
}
{
<insert_a_suppression_name_here>
exp-sgcheck:SorG
fun:ngx_vslprintf
fun:ngx_log_error_core
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_calloc
fun:ngx_event_process_init
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_malloc
fun:ngx_pcalloc
}
{
<insert_a_suppression_name_here>
Memcheck:Addr4
fun:lj_str_new
fun:lua_setfield
}
{
<insert_a_suppression_name_here>
Memcheck:Addr4
fun:lj_str_new
fun:lua_getfield
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:(below main)
}
{
<insert_a_suppression_name_here>
Memcheck:Param
epoll_ctl(event)
fun:epoll_ctl
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_event_process_init
}
{
<insert_a_suppression_name_here>
Memcheck:Cond
fun:ngx_conf_flush_files
fun:ngx_single_process_cycle
}
{
<insert_a_suppression_name_here>
Memcheck:Cond
fun:memcpy
fun:ngx_vslprintf
fun:ngx_log_error_core
fun:ngx_http_charset_header_filter
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:memalign
fun:posix_memalign
fun:ngx_memalign
fun:ngx_pcalloc
}
{
<insert_a_suppression_name_here>
Memcheck:Addr4
fun:lj_str_new
fun:lua_pushlstring
}
{
<insert_a_suppression_name_here>
Memcheck:Cond
fun:lj_str_new
fun:lj_str_fromnum
}
{
<insert_a_suppression_name_here>
Memcheck:Cond
fun:lj_str_new
fun:lua_pushlstring
}
{
<false_alarm_due_to_u32_alignment_in_luajit2>
Memcheck:Addr4
fun:lj_str_new
fun:lua_setfield
fun:ngx_http_lua_cache_store_code
}
{
<insert_a_suppression_name_here>
Memcheck:Cond
fun:lj_str_new
fun:lua_getfield
fun:ngx_http_lua_cache_load_code
}
{
<insert_a_suppression_name_here>
Memcheck:Cond
fun:lj_str_new
fun:lua_setfield
fun:ngx_http_lua_cache_store_code
}
{
<false_alarm_due_to_u32_alignment_in_luajit2>
Memcheck:Addr4
fun:lj_str_new
fun:lua_getfield
fun:ngx_http_lua_cache_load_code
}
{
<insert_a_suppression_name_here>
Memcheck:Param
socketcall.setsockopt(optval)
fun:setsockopt
fun:drizzle_state_connect
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_palloc_large
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_pool_cleanup_add
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_pnalloc
}
{
<insert_a_suppression_name_here>
Memcheck:Cond
fun:ngx_conf_flush_files
fun:ngx_single_process_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_palloc
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_pcalloc
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_malloc
fun:ngx_palloc_large
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_create_pool
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_malloc
fun:ngx_palloc
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_malloc
fun:ngx_pnalloc
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_palloc_large
fun:ngx_palloc
fun:ngx_array_push
fun:ngx_http_get_variable_index
fun:ngx_http_memc_add_variable
fun:ngx_http_memc_init
fun:ngx_http_block
fun:ngx_conf_parse
fun:ngx_init_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_event_process_init
fun:ngx_single_process_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_crc32_table_init
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_event_process_init
fun:ngx_worker_process_init
fun:ngx_worker_process_cycle
fun:ngx_spawn_process
fun:ngx_start_worker_processes
fun:ngx_master_process_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_palloc_large
fun:ngx_palloc
fun:ngx_pcalloc
fun:ngx_hash_init
fun:ngx_http_variables_init_vars
fun:ngx_http_block
fun:ngx_conf_parse
fun:ngx_init_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_palloc_large
fun:ngx_palloc
fun:ngx_pcalloc
fun:ngx_http_upstream_drizzle_create_srv_conf
fun:ngx_http_upstream
fun:ngx_conf_parse
fun:ngx_http_block
fun:ngx_conf_parse
fun:ngx_init_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_palloc_large
fun:ngx_palloc
fun:ngx_pcalloc
fun:ngx_hash_keys_array_init
fun:ngx_http_variables_add_core_vars
fun:ngx_http_core_preconfiguration
fun:ngx_http_block
fun:ngx_conf_parse
fun:ngx_init_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_palloc_large
fun:ngx_palloc
fun:ngx_array_push
fun:ngx_hash_add_key
fun:ngx_http_add_variable
fun:ngx_http_echo_add_variables
fun:ngx_http_echo_handler_init
fun:ngx_http_block
fun:ngx_conf_parse
fun:ngx_init_cycle
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_palloc_large
fun:ngx_palloc
fun:ngx_pcalloc
fun:ngx_http_upstream_drizzle_create_srv_conf
fun:ngx_http_core_server
fun:ngx_conf_parse
fun:ngx_http_block
fun:ngx_conf_parse
fun:ngx_init_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_palloc_large
fun:ngx_palloc
fun:ngx_pcalloc
fun:ngx_http_upstream_drizzle_create_srv_conf
fun:ngx_http_block
fun:ngx_conf_parse
fun:ngx_init_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_palloc_large
fun:ngx_palloc
fun:ngx_array_push
fun:ngx_hash_add_key
fun:ngx_http_variables_add_core_vars
fun:ngx_http_core_preconfiguration
fun:ngx_http_block
fun:ngx_conf_parse
fun:ngx_init_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_palloc_large
fun:ngx_palloc
fun:ngx_pcalloc
fun:ngx_init_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_palloc_large
fun:ngx_palloc
fun:ngx_hash_init
fun:ngx_http_upstream_init_main_conf
fun:ngx_http_block
fun:ngx_conf_parse
fun:ngx_init_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_palloc_large
fun:ngx_palloc
fun:ngx_pcalloc
fun:ngx_http_drizzle_keepalive_init
fun:ngx_http_upstream_drizzle_init
fun:ngx_http_upstream_init_main_conf
fun:ngx_http_block
fun:ngx_conf_parse
fun:ngx_init_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_palloc_large
fun:ngx_palloc
fun:ngx_hash_init
fun:ngx_http_variables_init_vars
fun:ngx_http_block
fun:ngx_conf_parse
fun:ngx_init_cycle
fun:main
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:memalign
fun:posix_memalign
fun:ngx_memalign
fun:ngx_create_pool
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:memalign
fun:posix_memalign
fun:ngx_memalign
fun:ngx_palloc_block
fun:ngx_palloc
}
{
<insert_a_suppression_name_here>
Memcheck:Cond
fun:index
fun:expand_dynamic_string_token
fun:_dl_map_object
fun:map_doit
fun:_dl_catch_error
fun:do_preload
fun:dl_main
fun:_dl_sysdep_start
fun:_dl_start
}

View file

@ -0,0 +1,2 @@
t/servroot/
t/error.log

View file

@ -0,0 +1,23 @@
Copyright (c) 2013, James Hurst
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View file

@ -0,0 +1,20 @@
OPENRESTY_PREFIX=/usr/local/openresty
PREFIX ?= /usr/local
LUA_INCLUDE_DIR ?= $(PREFIX)/include
LUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION)
INSTALL ?= install
TEST_FILE ?= t
.PHONY: all test install
all: ;
install: all
$(INSTALL) -d $(DESTDIR)/$(LUA_LIB_DIR)/resty
$(INSTALL) lib/resty/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/resty/
test: all
PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$$PATH TEST_NGINX_NO_SHUFFLE=1 prove -I../test-nginx/lib -r $(TEST_FILE)
util/lua-releng

View file

@ -0,0 +1,422 @@
# lua-resty-http
Lua HTTP client cosocket driver for [OpenResty](http://openresty.org/) / [ngx_lua](https://github.com/chaoslawful/lua-nginx-module).
# Status
Ready for testing. Probably production ready in most cases, though not yet proven in the wild. Please check the issues list and let me know if you have any problems / questions.
# Features
* HTTP 1.0 and 1.1
* Streaming interface to reading bodies using coroutines, for predictable memory usage in Lua land.
* Alternative simple interface for singleshot requests without manual connection step.
* Headers treated case insensitively.
* Chunked transfer encoding.
* Keepalive.
* Pipelining.
* Trailers.
# API
* [new](#name)
* [connect](#connect)
* [set_timeout](#set_timeout)
* [ssl_handshake](#ssl_handshake)
* [set_keepalive](#set_keepalive)
* [get_reused_times](#get_reused_times)
* [close](#close)
* [request](#request)
* [request_uri](#request_uri)
* [request_pipeline](#request_pipeline)
* [Response](#response)
* [body_reader](#resbody_reader)
* [read_body](#resread_body)
* [read_trailes](#resread_trailers)
* [Proxy](#proxy)
* [proxy_request](#proxy_request)
* [proxy_response](#proxy_response)
* [Utility](#utility)
* [parse_uri](#parse_uri)
* [get_client_body_reader](#get_client_body_reader)
## Synopsis
```` lua
lua_package_path "/path/to/lua-resty-http/lib/?.lua;;";
server {
location /simpleinterface {
resolver 8.8.8.8; # use Google's open DNS server for an example
content_by_lua '
-- For simple singleshot requests, use the URI interface.
local httpc = http.new()
local res, err = httpc:request_uri("http://example.com/helloworld", {
method = "POST",
body = "a=1&b=2",
headers = {
["Content-Type"] = "application/x-www-form-urlencoded",
}
})
if not res then
ngx.say("failed to request: ", err)
return
end
-- In this simple form, there is no manual connection step, so the body is read
-- all in one go, including any trailers, and the connection closed or keptalive
-- for you.
ngx.status = res.status
for k,v in pairs(res.headers) do
--
end
ngx.say(res.body)
';
}
location /genericinterface {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
-- The generic form gives us more control. We must connect manually.
httpc:set_timeout(500)
httpc:connect("127.0.0.1", 80)
-- And request using a path, rather than a full URI.
local res, err = httpc:request{
path = "/helloworld",
headers = {
["Host"] = "example.com",
},
}
if not res then
ngx.say("failed to request: ", err)
return
end
-- Now we can use the body_reader iterator, to stream the body according to our desired chunk size.
local reader = res.body_reader
repeat
local chunk, err = reader(8192)
if err then
ngx.log(ngx.ERR, err)
break
end
if chunk then
-- process
end
until not chunk
local ok, err = httpc:set_keepalive()
if not ok then
ngx.say("failed to set keepalive: ", err)
return
end
';
}
}
````
# Connection
## new
`syntax: httpc = http.new()`
Creates the http object. In case of failures, returns `nil` and a string describing the error.
## connect
`syntax: ok, err = httpc:connect(host, port, options_table?)`
`syntax: ok, err = httpc:connect("unix:/path/to/unix.sock", options_table?)`
Attempts to connect to the web server.
Before actually resolving the host name and connecting to the remote backend, this method will always look up the connection pool for matched idle connections created by previous calls of this method.
An optional Lua table can be specified as the last argument to this method to specify various connect options:
* `pool`
: Specifies a custom name for the connection pool being used. If omitted, then the connection pool name will be generated from the string template `<host>:<port>` or `<unix-socket-path>`.
## set_timeout
`syntax: httpc:set_timeout(time)`
Sets the timeout (in ms) protection for subsequent operations, including the `connect` method.
## ssl_handshake
`syntax: session, err = httpc:ssl_handshake(session, host, verify)`
Performs an SSL handshake on the TCP connection, only availble in ngx_lua > v0.9.11
See docs for [ngx.socket.tcp](https://github.com/openresty/lua-nginx-module#ngxsockettcp) for details.
## set_keepalive
`syntax: ok, err = httpc:set_keepalive(max_idle_timeout, pool_size)`
Attempts to puts the current connection into the ngx_lua cosocket connection pool.
You can specify the max idle timeout (in ms) when the connection is in the pool and the maximal size of the pool every nginx worker process.
Only call this method in the place you would have called the `close` method instead. Calling this method will immediately turn the current http object into the `closed` state. Any subsequent operations other than `connect()` on the current objet will return the `closed` error.
Note that calling this instead of `close` is "safe" in that it will conditionally close depending on the type of request. Specifically, a `1.0` request without `Connection: Keep-Alive` will be closed, as will a `1.1` request with `Connection: Close`.
In case of success, returns `1`. In case of errors, returns `nil, err`. In the case where the conneciton is conditionally closed as described above, returns `2` and the error string `connection must be closed`.
## get_reused_times
`syntax: times, err = httpc:get_reused_times()`
This method returns the (successfully) reused times for the current connection. In case of error, it returns `nil` and a string describing the error.
If the current connection does not come from the built-in connection pool, then this method always returns `0`, that is, the connection has never been reused (yet). If the connection comes from the connection pool, then the return value is always non-zero. So this method can also be used to determine if the current connection comes from the pool.
## close
`syntax: ok, err = http:close()`
Closes the current connection and returns the status.
In case of success, returns `1`. In case of errors, returns `nil` with a string describing the error.
# Requesting
## request
`syntax: res, err = httpc:request(params)`
Returns a `res` table or `nil` and an error message.
The `params` table accepts the following fields:
* `version` The HTTP version number, currently supporting 1.0 or 1.1.
* `method` The HTTP method string.
* `path` The path string.
* `headers` A table of request headers.
* `body` The request body as a string, or an iterator function (see [get_client_body_reader](#get_client_body_reader)).
* `ssl_verify` Verify SSL cert matches hostname
When the request is successful, `res` will contain the following fields:
* `status` The status code.
* `headers` A table of headers. Multiple headers with the same field name will be presented as a table of values.
* `has_body` A boolean flag indicating if there is a body to be read.
* `body_reader` An iterator function for reading the body in a streaming fashion.
* `read_body` A method to read the entire body into a string.
* `read_trailers` A method to merge any trailers underneath the headers, after reading the body.
## request_uri
`syntax: res, err = httpc:request_uri(uri, params)`
The simple interface. Options supplied in the `params` table are the same as in the generic interface, and will override components found in the uri itself.
In this mode, there is no need to connect manually first. The connection is made on your behalf, suiting cases where you simply need to grab a URI without too much hassle.
Additionally there is no ability to stream the response body in this mode. If the request is successful, `res` will contain the following fields:
* `status` The status code.
* `headers` A table of headers.
* `body` The response body as a string.
## request_pipeline
`syntax: responses, err = httpc:request_pipeline(params)`
This method works as per the [request](#request) method above, but `params` is instead a table of param tables. Each request is sent in order, and `responses` is returned as a table of response handles. For example:
```lua
local responses = httpc:request_pipeline{
{
path = "/b",
},
{
path = "/c",
},
{
path = "/d",
}
}
for i,r in ipairs(responses) do
if r.status then
ngx.say(r.status)
ngx.say(r:read_body())
end
end
```
Due to the nature of pipelining, no responses are actually read until you attempt to use the response fields (status / headers etc). And since the responses are read off in order, you must read the entire body (and any trailers if you have them), before attempting to read the next response.
Note this doesn't preclude the use of the streaming response body reader. Responses can still be streamed, so long as the entire body is streamed before attempting to access the next response.
Be sure to test at least one field (such as status) before trying to use the others, in case a socket read error has occurred.
# Response
## res.body_reader
The `body_reader` iterator can be used to stream the response body in chunk sizes of your choosing, as follows:
````lua
local reader = res.body_reader
repeat
local chunk, err = reader(8192)
if err then
ngx.log(ngx.ERR, err)
break
end
if chunk then
-- process
end
until not chunk
````
If the reader is called with no arguments, the behaviour depends on the type of connection. If the response is encoded as chunked, then the iterator will return the chunks as they arrive. If not, it will simply return the entire body.
Note that the size provided is actually a **maximum** size. So in the chunked transfer case, you may get chunks smaller than the size you ask, as a remainder of the actual HTTP chunks.
## res:read_body
`syntax: body, err = res:read_body()`
Reads the entire body into a local string.
## res:read_trailers
`syntax: res:read_trailers()`
This merges any trailers underneath the `res.headers` table itself. Must be called after reading the body.
# Proxy
There are two convenience methods for when one simply wishes to proxy the current request to the connected upstream, and safely send it downstream to the client, as a reverse proxy. A complete example:
```lua
local http = require "resty.http"
local httpc = http.new()
httpc:set_timeout(500)
local ok, err = httpc:connect(HOST, PORT)
if not ok then
ngx.log(ngx.ERR, err)
return
end
httpc:set_timeout(2000)
httpc:proxy_response(httpc:proxy_request())
httpc:set_keepalive()
```
## proxy_request
`syntax: local res, err = httpc:proxy_request(request_body_chunk_size?)`
Performs a request using the current client request arguments, effectively proxying to the connected upstream. The request body will be read in a streaming fashion, according to `request_body_chunk_size` (see [documentation on the client body reader](#get_client_body_reader) below).
## proxy_response
`syntax: httpc:proxy_response(res, chunksize?)`
Sets the current response based on the given `res`. Ensures that hop-by-hop headers are not sent downstream, and will read the response according to `chunksize` (see [documentation on the body reader](#resbody_reader) above).
# Utility
## parse_uri
`syntax: local scheme, host, port, path = unpack(httpc:parse_uri(uri))`
This is a convenience function allowing one to more easily use the generic interface, when the input data is a URI.
## get_client_body_reader
`syntax: reader, err = httpc:get_client_body_reader(chunksize?, sock?)`
Returns an iterator function which can be used to read the downstream client request body in a streaming fashion. You may also specify an optional default chunksize (default is `65536`), or an already established socket in
place of the client request.
Example:
```lua
local req_reader = httpc:get_client_body_reader()
repeat
local chunk, err = req_reader(8192)
if err then
ngx.log(ngx.ERR, err)
break
end
if chunk then
-- process
end
until not chunk
```
This iterator can also be used as the value for the body field in request params, allowing one to stream the request body into a proxied upstream request.
```lua
local client_body_reader, err = httpc:get_client_body_reader()
local res, err = httpc:request{
path = "/helloworld",
body = client_body_reader,
}
```
If `sock` is specified,
# Author
James Hurst <james@pintsized.co.uk>
Originally started life based on https://github.com/bakins/lua-resty-http-simple. Cosocket docs and implementation borrowed from the other lua-resty-* cosocket modules.
# Licence
This module is licensed under the 2-clause BSD license.
Copyright (c) 2013, James Hurst <james@pintsized.co.uk>
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View file

@ -0,0 +1,814 @@
local http_headers = require "resty.http_headers"
local ngx_socket_tcp = ngx.socket.tcp
local ngx_req = ngx.req
local ngx_req_socket = ngx_req.socket
local ngx_req_get_headers = ngx_req.get_headers
local ngx_req_get_method = ngx_req.get_method
local str_gmatch = string.gmatch
local str_lower = string.lower
local str_upper = string.upper
local str_find = string.find
local str_sub = string.sub
local str_gsub = string.gsub
local tbl_concat = table.concat
local tbl_insert = table.insert
local ngx_encode_args = ngx.encode_args
local ngx_re_match = ngx.re.match
local ngx_re_gsub = ngx.re.gsub
local ngx_log = ngx.log
local ngx_DEBUG = ngx.DEBUG
local ngx_ERR = ngx.ERR
local ngx_NOTICE = ngx.NOTICE
local ngx_var = ngx.var
local co_yield = coroutine.yield
local co_create = coroutine.create
local co_status = coroutine.status
local co_resume = coroutine.resume
-- http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.5.1
local HOP_BY_HOP_HEADERS = {
["connection"] = true,
["keep-alive"] = true,
["proxy-authenticate"] = true,
["proxy-authorization"] = true,
["te"] = true,
["trailers"] = true,
["transfer-encoding"] = true,
["upgrade"] = true,
["content-length"] = true, -- Not strictly hop-by-hop, but Nginx will deal
-- with this (may send chunked for example).
}
-- Reimplemented coroutine.wrap, returning "nil, err" if the coroutine cannot
-- be resumed. This protects user code from inifite loops when doing things like
-- repeat
-- local chunk, err = res.body_reader()
-- if chunk then -- <-- This could be a string msg in the core wrap function.
-- ...
-- end
-- until not chunk
local co_wrap = function(func)
local co = co_create(func)
if not co then
return nil, "could not create coroutine"
else
return function(...)
if co_status(co) == "suspended" then
return select(2, co_resume(co, ...))
else
return nil, "can't resume a " .. co_status(co) .. " coroutine"
end
end
end
end
local _M = {
_VERSION = '0.06',
}
_M._USER_AGENT = "lua-resty-http/" .. _M._VERSION .. " (Lua) ngx_lua/" .. ngx.config.ngx_lua_version
local mt = { __index = _M }
local HTTP = {
[1.0] = " HTTP/1.0\r\n",
[1.1] = " HTTP/1.1\r\n",
}
local DEFAULT_PARAMS = {
method = "GET",
path = "/",
version = 1.1,
}
function _M.new(self)
local sock, err = ngx_socket_tcp()
if not sock then
return nil, err
end
return setmetatable({ sock = sock, keepalive = true }, mt)
end
function _M.set_timeout(self, timeout)
local sock = self.sock
if not sock then
return nil, "not initialized"
end
return sock:settimeout(timeout)
end
function _M.ssl_handshake(self, ...)
local sock = self.sock
if not sock then
return nil, "not initialized"
end
return sock:sslhandshake(...)
end
function _M.connect(self, ...)
local sock = self.sock
if not sock then
return nil, "not initialized"
end
self.host = select(1, ...)
self.keepalive = true
return sock:connect(...)
end
function _M.set_keepalive(self, ...)
local sock = self.sock
if not sock then
return nil, "not initialized"
end
if self.keepalive == true then
return sock:setkeepalive(...)
else
-- The server said we must close the connection, so we cannot setkeepalive.
-- If close() succeeds we return 2 instead of 1, to differentiate between
-- a normal setkeepalive() failure and an intentional close().
local res, err = sock:close()
if res then
return 2, "connection must be closed"
else
return res, err
end
end
end
function _M.get_reused_times(self)
local sock = self.sock
if not sock then
return nil, "not initialized"
end
return sock:getreusedtimes()
end
function _M.close(self)
local sock = self.sock
if not sock then
return nil, "not initialized"
end
return sock:close()
end
local function _should_receive_body(method, code)
if method == "HEAD" then return nil end
if code == 204 or code == 304 then return nil end
if code >= 100 and code < 200 then return nil end
return true
end
function _M.parse_uri(self, uri)
local m, err = ngx_re_match(uri, [[^(http[s]*)://([^:/]+)(?::(\d+))?(.*)]],
"jo")
if not m then
if err then
return nil, "failed to match the uri: " .. err
end
return nil, "bad uri"
else
if not m[3] then
if m[1] == "https" then
m[3] = 443
else
m[3] = 80
end
end
if not m[4] then m[4] = "/" end
return m, nil
end
end
local function _format_request(params)
local version = params.version
local headers = params.headers or {}
local query = params.query or ""
if query then
if type(query) == "table" then
query = "?" .. ngx_encode_args(query)
end
end
-- Initialize request
local req = {
str_upper(params.method),
" ",
params.path,
query,
HTTP[version],
-- Pre-allocate slots for minimum headers and carriage return.
true,
true,
true,
}
local c = 6 -- req table index it's faster to do this inline vs table.insert
-- Append headers
for key, values in pairs(headers) do
if type(values) ~= "table" then
values = {values}
end
key = tostring(key)
for _, value in pairs(values) do
req[c] = key .. ": " .. tostring(value) .. "\r\n"
c = c + 1
end
end
-- Close headers
req[c] = "\r\n"
return tbl_concat(req)
end
local function _receive_status(sock)
local line, err = sock:receive("*l")
if not line then
return nil, nil, err
end
return tonumber(str_sub(line, 10, 12)), tonumber(str_sub(line, 6, 8))
end
local function _receive_headers(sock)
local headers = http_headers.new()
repeat
local line, err = sock:receive("*l")
if not line then
return nil, err
end
for key, val in str_gmatch(line, "([^:%s]+):%s*(.+)") do
if headers[key] then
if type(headers[key]) ~= "table" then
headers[key] = { headers[key] }
end
tbl_insert(headers[key], tostring(val))
else
headers[key] = tostring(val)
end
end
until str_find(line, "^%s*$")
return headers, nil
end
local function _chunked_body_reader(sock, default_chunk_size)
return co_wrap(function(max_chunk_size)
local max_chunk_size = max_chunk_size or default_chunk_size
local remaining = 0
local length
repeat
-- If we still have data on this chunk
if max_chunk_size and remaining > 0 then
if remaining > max_chunk_size then
-- Consume up to max_chunk_size
length = max_chunk_size
remaining = remaining - max_chunk_size
else
-- Consume all remaining
length = remaining
remaining = 0
end
else -- This is a fresh chunk
-- Receive the chunk size
local str, err = sock:receive("*l")
if not str then
co_yield(nil, err)
end
length = tonumber(str, 16)
if not length then
co_yield(nil, "unable to read chunksize")
end
if max_chunk_size and length > max_chunk_size then
-- Consume up to max_chunk_size
remaining = length - max_chunk_size
length = max_chunk_size
end
end
if length > 0 then
local str, err = sock:receive(length)
if not str then
co_yield(nil, err)
end
max_chunk_size = co_yield(str) or default_chunk_size
-- If we're finished with this chunk, read the carriage return.
if remaining == 0 then
sock:receive(2) -- read \r\n
end
else
-- Read the last (zero length) chunk's carriage return
sock:receive(2) -- read \r\n
end
until length == 0
end)
end
local function _body_reader(sock, content_length, default_chunk_size)
return co_wrap(function(max_chunk_size)
local max_chunk_size = max_chunk_size or default_chunk_size
if not content_length and max_chunk_size then
-- We have no length, but wish to stream.
-- HTTP 1.0 with no length will close connection, so read chunks to the end.
repeat
local str, err, partial = sock:receive(max_chunk_size)
if not str and err == "closed" then
max_chunk_size = tonumber(co_yield(partial, err) or default_chunk_size)
end
max_chunk_size = tonumber(co_yield(str) or default_chunk_size)
if max_chunk_size and max_chunk_size < 0 then max_chunk_size = nil end
if not max_chunk_size then
ngx_log(ngx_ERR, "Buffer size not specified, bailing")
break
end
until not str
elseif not content_length then
-- We have no length but don't wish to stream.
-- HTTP 1.0 with no length will close connection, so read to the end.
co_yield(sock:receive("*a"))
elseif not max_chunk_size then
-- We have a length and potentially keep-alive, but want everything.
co_yield(sock:receive(content_length))
else
-- We have a length and potentially a keep-alive, and wish to stream
-- the response.
local received = 0
repeat
local length = max_chunk_size
if received + length > content_length then
length = content_length - received
end
if length > 0 then
local str, err = sock:receive(length)
if not str then
max_chunk_size = tonumber(co_yield(nil, err) or default_chunk_size)
end
received = received + length
max_chunk_size = tonumber(co_yield(str) or default_chunk_size)
if max_chunk_size and max_chunk_size < 0 then max_chunk_size = nil end
if not max_chunk_size then
ngx_log(ngx_ERR, "Buffer size not specified, bailing")
break
end
end
until length == 0
end
end)
end
local function _no_body_reader()
return nil
end
local function _read_body(res)
local reader = res.body_reader
if not reader then
-- Most likely HEAD or 304 etc.
return nil, "no body to be read"
end
local chunks = {}
local c = 1
local chunk, err
repeat
chunk, err = reader()
if err then
return nil, err, tbl_concat(chunks) -- Return any data so far.
end
if chunk then
chunks[c] = chunk
c = c + 1
end
until not chunk
return tbl_concat(chunks)
end
local function _trailer_reader(sock)
return co_wrap(function()
co_yield(_receive_headers(sock))
end)
end
local function _read_trailers(res)
local reader = res.trailer_reader
if not reader then
return nil, "no trailers"
end
local trailers = reader()
setmetatable(res.headers, { __index = trailers })
end
local function _send_body(sock, body)
if type(body) == 'function' then
repeat
local chunk, err, partial = body()
if chunk then
local ok,err = sock:send(chunk)
if not ok then
return nil, err
end
elseif err ~= nil then
return nil, err, partial
end
until chunk == nil
elseif body ~= nil then
local bytes, err = sock:send(body)
if not bytes then
return nil, err
end
end
return true, nil
end
local function _handle_continue(sock, body)
local status, version, err = _receive_status(sock)
if not status then
return nil, err
end
-- Only send body if we receive a 100 Continue
if status == 100 then
local ok, err = sock:receive("*l") -- Read carriage return
if not ok then
return nil, err
end
_send_body(sock, body)
end
return status, version, err
end
function _M.send_request(self, params)
-- Apply defaults
setmetatable(params, { __index = DEFAULT_PARAMS })
local sock = self.sock
local body = params.body
local headers = http_headers.new()
local params_headers = params.headers
if params_headers then
-- We assign one by one so that the metatable can handle case insensitivity
-- for us. You can blame the spec for this inefficiency.
for k,v in pairs(params_headers) do
headers[k] = v
end
end
-- Ensure minimal headers are set
if type(body) == 'string' and not headers["Content-Length"] then
headers["Content-Length"] = #body
end
if not headers["Host"] then
headers["Host"] = self.host
end
if not headers["User-Agent"] then
headers["User-Agent"] = _M._USER_AGENT
end
if params.version == 1.0 and not headers["Connection"] then
headers["Connection"] = "Keep-Alive"
end
params.headers = headers
-- Format and send request
local req = _format_request(params)
ngx_log(ngx_DEBUG, "\n", req)
local bytes, err = sock:send(req)
if not bytes then
return nil, err
end
-- Send the request body, unless we expect: continue, in which case
-- we handle this as part of reading the response.
if headers["Expect"] ~= "100-continue" then
local ok, err, partial = _send_body(sock, body)
if not ok then
return nil, err, partial
end
end
return true
end
function _M.read_response(self, params)
local sock = self.sock
local status, version, err
-- If we expect: continue, we need to handle this, sending the body if allowed.
-- If we don't get 100 back, then status is the actual status.
if params.headers["Expect"] == "100-continue" then
local _status, _version, _err = _handle_continue(sock, params.body)
if not _status then
return nil, _err
elseif _status ~= 100 then
status, version, err = _status, _version, _err
end
end
-- Just read the status as normal.
if not status then
status, version, err = _receive_status(sock)
if not status then
return nil, err
end
end
local res_headers, err = _receive_headers(sock)
if not res_headers then
return nil, err
end
-- Determine if we should keepalive or not.
local ok, connection = pcall(str_lower, res_headers["Connection"])
if ok then
if (version == 1.1 and connection == "close") or
(version == 1.0 and connection ~= "keep-alive") then
self.keepalive = false
end
end
local body_reader = _no_body_reader
local trailer_reader, err = nil, nil
local has_body = false
-- Receive the body_reader
if _should_receive_body(params.method, status) then
local ok, encoding = pcall(str_lower, res_headers["Transfer-Encoding"])
if ok and version == 1.1 and encoding == "chunked" then
body_reader, err = _chunked_body_reader(sock)
has_body = true
else
local ok, length = pcall(tonumber, res_headers["Content-Length"])
if ok then
body_reader, err = _body_reader(sock, length)
has_body = true
end
end
end
if res_headers["Trailer"] then
trailer_reader, err = _trailer_reader(sock)
end
if err then
return nil, err
else
return {
status = status,
headers = res_headers,
has_body = has_body,
body_reader = body_reader,
read_body = _read_body,
trailer_reader = trailer_reader,
read_trailers = _read_trailers,
}
end
end
function _M.request(self, params)
local res, err = self:send_request(params)
if not res then
return res, err
else
return self:read_response(params)
end
end
function _M.request_pipeline(self, requests)
for i, params in ipairs(requests) do
if params.headers and params.headers["Expect"] == "100-continue" then
return nil, "Cannot pipeline request specifying Expect: 100-continue"
end
local res, err = self:send_request(params)
if not res then
return res, err
end
end
local responses = {}
for i, params in ipairs(requests) do
responses[i] = setmetatable({
params = params,
response_read = false,
}, {
-- Read each actual response lazily, at the point the user tries
-- to access any of the fields.
__index = function(t, k)
local res, err
if t.response_read == false then
res, err = _M.read_response(self, t.params)
t.response_read = true
if not res then
ngx_log(ngx_ERR, err)
else
for rk, rv in pairs(res) do
t[rk] = rv
end
end
end
return rawget(t, k)
end,
})
end
return responses
end
function _M.request_uri(self, uri, params)
if not params then params = {} end
local parsed_uri, err = self:parse_uri(uri)
if not parsed_uri then
return nil, err
end
local scheme, host, port, path = unpack(parsed_uri)
if not params.path then params.path = path end
local c, err = self:connect(host, port)
if not c then
return nil, err
end
if scheme == "https" then
local verify = true
if params.ssl_verify == false then
verify = false
end
local ok, err = self:ssl_handshake(nil, host, verify)
if not ok then
return nil, err
end
end
local res, err = self:request(params)
if not res then
return nil, err
end
local body, err = res:read_body()
if not body then
return nil, err
end
res.body = body
local ok, err = self:set_keepalive()
if not ok then
ngx_log(ngx_ERR, err)
end
return res, nil
end
function _M.get_client_body_reader(self, chunksize, sock)
local chunksize = chunksize or 65536
if not sock then
local ok, err
ok, sock, err = pcall(ngx_req_socket)
if not ok then
return nil, sock -- pcall err
end
if not sock then
if err == "no body" then
return nil
else
return nil, err
end
end
end
local headers = ngx_req_get_headers()
local length = headers.content_length
local encoding = headers.transfer_encoding
if length then
return _body_reader(sock, tonumber(length), chunksize)
elseif encoding and str_lower(encoding) == 'chunked' then
-- Not yet supported by ngx_lua but should just work...
return _chunked_body_reader(sock, chunksize)
else
return nil
end
end
function _M.proxy_request(self, chunksize)
return self:request{
method = ngx_req_get_method(),
path = ngx_re_gsub(ngx_var.uri, "\\s", "%20", "jo") .. ngx_var.is_args .. (ngx_var.query_string or ""),
body = self:get_client_body_reader(chunksize),
headers = ngx_req_get_headers(),
}
end
function _M.proxy_response(self, response, chunksize)
if not response then
ngx_log(ngx_ERR, "no response provided")
return
end
ngx.status = response.status
-- Filter out hop-by-hop headeres
for k,v in pairs(response.headers) do
if not HOP_BY_HOP_HEADERS[str_lower(k)] then
ngx.header[k] = v
end
end
local reader = response.body_reader
repeat
local chunk, err = reader(chunksize)
if err then
ngx_log(ngx_ERR, err)
break
end
if chunk then
ngx.print(chunk)
end
until not chunk
end
return _M

View file

@ -0,0 +1,62 @@
local rawget, rawset, setmetatable =
rawget, rawset, setmetatable
local str_gsub = string.gsub
local str_lower = string.lower
local _M = {
_VERSION = '0.01',
}
-- Returns an empty headers table with internalised case normalisation.
-- Supports the same cases as in ngx_lua:
--
-- headers.content_length
-- headers["content-length"]
-- headers["Content-Length"]
function _M.new(self)
local mt = {
normalised = {},
}
mt.__index = function(t, k)
local k_hyphened = str_gsub(k, "_", "-")
local matched = rawget(t, k)
if matched then
return matched
else
local k_normalised = str_lower(k_hyphened)
return rawget(t, mt.normalised[k_normalised])
end
end
-- First check the normalised table. If there's no match (first time) add an entry for
-- our current case in the normalised table. This is to preserve the human (prettier) case
-- instead of outputting lowercased header names.
--
-- If there's a match, we're being updated, just with a different case for the key. We use
-- the normalised table to give us the original key, and perorm a rawset().
mt.__newindex = function(t, k, v)
-- we support underscore syntax, so always hyphenate.
local k_hyphened = str_gsub(k, "_", "-")
-- lowercase hyphenated is "normalised"
local k_normalised = str_lower(k_hyphened)
if not mt.normalised[k_normalised] then
mt.normalised[k_normalised] = k_hyphened
rawset(t, k_hyphened, v)
else
rawset(t, mt.normalised[k_normalised], v)
end
end
return setmetatable({}, mt)
end
return _M

View file

@ -0,0 +1,33 @@
package = "lua-resty-http"
version = "0.06-0"
source = {
url = "git://github.com/pintsized/lua-resty-http",
tag = "v0.06"
}
description = {
summary = "Lua HTTP client cosocket driver for OpenResty / ngx_lua.",
detailed = [[
Features an HTTP 1.0 and 1.1 streaming interface to reading
bodies using coroutines, for predictable memory usage in Lua
land. Alternative simple interface for singleshot requests
without manual connection step. Supports chunked transfer
encoding, keepalive, pipelining, and trailers. Headers are
treated case insensitively. Probably production ready in most
cases, though not yet proven in the wild.
Recommended by the OpenResty maintainer as a long-term
replacement for internal requests through ngx.location.capture.
]],
homepage = "https://github.com/pintsized/lua-resty-http",
license = "2-clause BSD",
maintainer = "James Hurst <james@pintsized.co.uk>"
}
dependencies = {
"lua >= 5.1",
}
build = {
type = "builtin",
modules = {
["resty.http"] = "lib/resty/http.lua",
["resty.http_headers"] = "lib/resty/http_headers.lua"
}
}

View file

@ -0,0 +1,231 @@
# vim:set ft= ts=4 sw=4 et:
use Test::Nginx::Socket;
use Cwd qw(cwd);
plan tests => repeat_each() * (blocks() * 4) + 1;
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
error_log logs/error.log debug;
};
$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
no_long_string();
#no_diff();
run_tests();
__DATA__
=== TEST 1: Simple default get.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b"
}
ngx.status = res.status
ngx.print(res:read_body())
httpc:close()
';
}
location = /b {
echo "OK";
}
--- request
GET /a
--- response_body
OK
--- no_error_log
[error]
[warn]
=== TEST 2: HTTP 1.0
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
version = 1.0,
path = "/b"
}
ngx.status = res.status
ngx.print(res:read_body())
httpc:close()
';
}
location = /b {
echo "OK";
}
--- request
GET /a
--- response_body
OK
--- no_error_log
[error]
[warn]
=== TEST 3: Status code
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b"
}
ngx.status = res.status
ngx.print(res:read_body())
httpc:close()
';
}
location = /b {
content_by_lua '
ngx.status = 404
ngx.say("OK")
';
}
--- request
GET /a
--- response_body
OK
--- error_code: 404
--- no_error_log
[error]
[warn]
=== TEST 4: Response headers
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b"
}
ngx.status = res.status
ngx.say(res.headers["X-Test"])
httpc:close()
';
}
location = /b {
content_by_lua '
ngx.header["X-Test"] = "x-value"
ngx.say("OK")
';
}
--- request
GET /a
--- response_body
x-value
--- no_error_log
[error]
[warn]
=== TEST 5: Query
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
query = {
a = 1,
b = 2,
},
path = "/b"
}
ngx.status = res.status
for k,v in pairs(res.headers) do
ngx.header[k] = v
end
ngx.print(res:read_body())
httpc:close()
';
}
location = /b {
content_by_lua '
for k,v in pairs(ngx.req.get_uri_args()) do
ngx.header["X-Header-" .. string.upper(k)] = v
end
';
}
--- request
GET /a
--- response_headers
X-Header-A: 1
X-Header-B: 2
--- no_error_log
[error]
[warn]
=== TEST 7: HEAD has no body.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
method = "HEAD",
path = "/b"
}
local body = res:read_body()
if body then
ngx.print(body)
end
httpc:close()
';
}
location = /b {
echo "OK";
}
--- request
GET /a
--- response_body
--- no_error_log
[error]
[warn]

View file

@ -0,0 +1,158 @@
# vim:set ft= ts=4 sw=4 et:
use Test::Nginx::Socket;
use Cwd qw(cwd);
plan tests => repeat_each() * (blocks() * 4);
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
};
$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
no_long_string();
#no_diff();
run_tests();
__DATA__
=== TEST 1: Non chunked.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b"
}
local body = res:read_body()
ngx.say(#body)
httpc:close()
';
}
location = /b {
chunked_transfer_encoding off;
content_by_lua '
local len = 32768
local t = {}
for i=1,len do
t[i] = 0
end
ngx.print(table.concat(t))
';
}
--- request
GET /a
--- response_body
32768
--- no_error_log
[error]
[warn]
=== TEST 2: Chunked. The number of chunks received when no max size is given proves the response was in fact chunked.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b"
}
local chunks = {}
local c = 1
repeat
local chunk, err = res.body_reader()
if chunk then
chunks[c] = chunk
c = c + 1
end
until not chunk
local body = table.concat(chunks)
ngx.say(#body)
ngx.say(#chunks)
httpc:close()
';
}
location = /b {
content_by_lua '
local len = 32768
local t = {}
for i=1,len do
t[i] = 0
end
ngx.print(table.concat(t))
local len = 32768
local t = {}
for i=1,len do
t[i] = 0
end
ngx.print(table.concat(t))
';
}
--- request
GET /a
--- response_body
65536
2
--- no_error_log
[error]
[warn]
=== TEST 3: Chunked using read_body method.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b"
}
local body = res:read_body()
ngx.say(#body)
httpc:close()
';
}
location = /b {
content_by_lua '
local len = 32768
local t = {}
for i=1,len do
t[i] = 0
end
ngx.print(table.concat(t))
local len = 32768
local t = {}
for i=1,len do
t[i] = 0
end
ngx.print(table.concat(t))
';
}
--- request
GET /a
--- response_body
65536
--- no_error_log
[error]
[warn]

View file

@ -0,0 +1,185 @@
# vim:set ft= ts=4 sw=4 et:
use Test::Nginx::Socket;
use Cwd qw(cwd);
plan tests => repeat_each() * (blocks() * 4);
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
};
$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
no_long_string();
#no_diff();
run_tests();
__DATA__
=== TEST 1: POST form-urlencoded
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
body = "a=1&b=2&c=3",
path = "/b",
headers = {
["Content-Type"] = "application/x-www-form-urlencoded",
}
}
ngx.say(res:read_body())
httpc:close()
';
}
location = /b {
content_by_lua '
ngx.req.read_body()
local args = ngx.req.get_post_args()
ngx.say("a: ", args.a)
ngx.say("b: ", args.b)
ngx.print("c: ", args.c)
';
}
--- request
GET /a
--- response_body
a: 1
b: 2
c: 3
--- no_error_log
[error]
[warn]
=== TEST 2: POST form-urlencoded 1.0
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
method = "POST",
body = "a=1&b=2&c=3",
path = "/b",
headers = {
["Content-Type"] = "application/x-www-form-urlencoded",
},
version = 1.0,
}
ngx.say(res:read_body())
httpc:close()
';
}
location = /b {
content_by_lua '
ngx.req.read_body()
local args = ngx.req.get_post_args()
ngx.say(ngx.req.get_method())
ngx.say("a: ", args.a)
ngx.say("b: ", args.b)
ngx.print("c: ", args.c)
';
}
--- request
GET /a
--- response_body
POST
a: 1
b: 2
c: 3
--- no_error_log
[error]
[warn]
=== TEST 3: 100 Continue does not end requset
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
body = "a=1&b=2&c=3",
path = "/b",
headers = {
["Expect"] = "100-continue",
["Content-Type"] = "application/x-www-form-urlencoded",
}
}
ngx.say(res.status)
ngx.say(res:read_body())
httpc:close()
';
}
location = /b {
content_by_lua '
ngx.req.read_body()
local args = ngx.req.get_post_args()
ngx.say("a: ", args.a)
ngx.say("b: ", args.b)
ngx.print("c: ", args.c)
';
}
--- request
GET /a
--- response_body
200
a: 1
b: 2
c: 3
--- no_error_log
[error]
[warn]
=== TEST 4: Return non-100 status to user
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b",
headers = {
["Expect"] = "100-continue",
["Content-Type"] = "application/x-www-form-urlencoded",
}
}
if not res then
ngx.say(err)
end
ngx.say(res.status)
ngx.say(res:read_body())
httpc:close()
';
}
location = /b {
return 417 "Expectation Failed";
}
--- request
GET /a
--- response_body
417
Expectation Failed
--- no_error_log
[error]
[warn]

View file

@ -0,0 +1,151 @@
# vim:set ft= ts=4 sw=4 et:
use Test::Nginx::Socket;
use Cwd qw(cwd);
plan tests => repeat_each() * (blocks() * 4);
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
};
$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
no_long_string();
#no_diff();
run_tests();
__DATA__
=== TEST 1: Trailers. Check Content-MD5 generated after the body is sent matches up.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b",
headers = {
["TE"] = "trailers",
}
}
local body = res:read_body()
local hash = ngx.md5(body)
res:read_trailers()
if res.headers["Content-MD5"] == hash then
ngx.say("OK")
else
ngx.say(res.headers["Content-MD5"])
end
';
}
location = /b {
content_by_lua '
-- We use the raw socket to compose a response, since OpenResty
-- doesnt support trailers natively.
ngx.req.read_body()
local sock, err = ngx.req.socket(true)
if not sock then
ngx.say(err)
end
local res = {}
table.insert(res, "HTTP/1.1 200 OK")
table.insert(res, "Date: " .. ngx.http_time(ngx.time()))
table.insert(res, "Transfer-Encoding: chunked")
table.insert(res, "Trailer: Content-MD5")
table.insert(res, "")
local body = "Hello, World"
table.insert(res, string.format("%x", #body))
table.insert(res, body)
table.insert(res, "0")
table.insert(res, "")
table.insert(res, "Content-MD5: " .. ngx.md5(body))
table.insert(res, "")
table.insert(res, "")
sock:send(table.concat(res, "\\r\\n"))
';
}
--- request
GET /a
--- response_body
OK
--- no_error_log
[error]
[warn]
=== TEST 2: Advertised trailer does not exist, handled gracefully.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b",
headers = {
["TE"] = "trailers",
}
}
local body = res:read_body()
local hash = ngx.md5(body)
res:read_trailers()
ngx.say("OK")
httpc:close()
';
}
location = /b {
content_by_lua '
-- We use the raw socket to compose a response, since OpenResty
-- doesnt support trailers natively.
ngx.req.read_body()
local sock, err = ngx.req.socket(true)
if not sock then
ngx.say(err)
end
local res = {}
table.insert(res, "HTTP/1.1 200 OK")
table.insert(res, "Date: " .. ngx.http_time(ngx.time()))
table.insert(res, "Transfer-Encoding: chunked")
table.insert(res, "Trailer: Content-MD5")
table.insert(res, "")
local body = "Hello, World"
table.insert(res, string.format("%x", #body))
table.insert(res, body)
table.insert(res, "0")
table.insert(res, "")
table.insert(res, "")
sock:send(table.concat(res, "\\r\\n"))
';
}
--- request
GET /a
--- response_body
OK
--- no_error_log
[error]
[warn]

View file

@ -0,0 +1,566 @@
# vim:set ft= ts=4 sw=4 et:
use Test::Nginx::Socket;
use Cwd qw(cwd);
plan tests => repeat_each() * (blocks() * 4) - 1;
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
};
$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
no_long_string();
#no_diff();
run_tests();
__DATA__
=== TEST 1: Chunked streaming body reader returns the right content length.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b",
}
local chunks = {}
repeat
local chunk = res.body_reader()
if chunk then
table.insert(chunks, chunk)
end
until not chunk
local body = table.concat(chunks)
ngx.say(#body)
ngx.say(res.headers["Transfer-Encoding"])
httpc:close()
';
}
location = /b {
content_by_lua '
local len = 32768
local t = {}
for i=1,len do
t[i] = 0
end
ngx.print(table.concat(t))
';
}
--- request
GET /a
--- response_body
32768
chunked
--- no_error_log
[error]
[warn]
=== TEST 2: Non-Chunked streaming body reader returns the right content length.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b",
}
local chunks = {}
repeat
local chunk = res.body_reader()
if chunk then
table.insert(chunks, chunk)
end
until not chunk
local body = table.concat(chunks)
ngx.say(#body)
ngx.say(res.headers["Transfer-Encoding"])
ngx.say(#chunks)
httpc:close()
';
}
location = /b {
chunked_transfer_encoding off;
content_by_lua '
local len = 32768
local t = {}
for i=1,len do
t[i] = 0
end
ngx.print(table.concat(t))
';
}
--- request
GET /a
--- response_body
32768
nil
1
--- no_error_log
[error]
[warn]
=== TEST 2b: Non-Chunked streaming body reader, buffer size becomes nil
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b",
}
local chunks = {}
local buffer_size = 16384
repeat
local chunk = res.body_reader(buffer_size)
if chunk then
table.insert(chunks, chunk)
end
buffer_size = nil
until not chunk
local body = table.concat(chunks)
ngx.say(res.headers["Transfer-Encoding"])
httpc:close()
';
}
location = /b {
chunked_transfer_encoding off;
content_by_lua '
local len = 32768
local t = {}
for i=1,len do
t[i] = 0
end
ngx.print(table.concat(t))
';
}
--- request
GET /a
--- response_body
nil
--- error_log
Buffer size not specified, bailing
=== TEST 3: HTTP 1.0 body reader with no max size returns the right content length.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b",
version = 1.0,
}
local chunks = {}
repeat
local chunk = res.body_reader()
if chunk then
table.insert(chunks, chunk)
end
until not chunk
local body = table.concat(chunks)
ngx.say(#body)
ngx.say(res.headers["Transfer-Encoding"])
ngx.say(#chunks)
httpc:close()
';
}
location = /b {
chunked_transfer_encoding off;
content_by_lua '
local len = 32768
local t = {}
for i=1,len do
t[i] = 0
end
ngx.print(table.concat(t))
';
}
--- request
GET /a
--- response_body
32768
nil
1
--- no_error_log
[error]
[warn]
=== TEST 4: HTTP 1.0 body reader with max chunk size returns the right content length.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b",
version = 1.0,
}
local chunks = {}
local size = 8192
repeat
local chunk = res.body_reader(size)
if chunk then
table.insert(chunks, chunk)
end
size = size + size
until not chunk
local body = table.concat(chunks)
ngx.say(#body)
ngx.say(res.headers["Transfer-Encoding"])
ngx.say(#chunks)
httpc:close()
';
}
location = /b {
chunked_transfer_encoding off;
content_by_lua '
local len = 32769
local t = {}
for i=1,len do
t[i] = 0
end
ngx.print(table.concat(t))
';
}
--- request
GET /a
--- response_body
32769
nil
3
--- no_error_log
[error]
[warn]
=== TEST 4b: HTTP 1.0 body reader with no content length, stream works as expected.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b",
version = 1.0,
}
local chunks = {}
local size = 8192
repeat
local chunk = res.body_reader(size)
if chunk then
table.insert(chunks, chunk)
end
size = size + size
until not chunk
local body = table.concat(chunks)
ngx.say(#body)
ngx.say(#chunks)
httpc:close()
';
}
location = /b {
content_by_lua '
ngx.req.read_body()
local sock, err = ngx.req.socket(true)
if not sock then
ngx.say(err)
end
local res = {}
table.insert(res, "HTTP/1.0 200 OK")
table.insert(res, "Date: " .. ngx.http_time(ngx.time()))
table.insert(res, "")
local len = 32769
local t = {}
for i=1,len do
t[i] = 0
end
table.insert(res, table.concat(t))
sock:send(table.concat(res, "\\r\\n"))
';
}
--- request
GET /a
--- response_body
32769
3
--- no_error_log
[error]
[warn]
=== TEST 5: Chunked streaming body reader with max chunk size returns the right content length.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b",
}
local chunks = {}
local size = 8192
repeat
local chunk = res.body_reader(size)
if chunk then
table.insert(chunks, chunk)
end
size = size + size
until not chunk
local body = table.concat(chunks)
ngx.say(#body)
ngx.say(res.headers["Transfer-Encoding"])
ngx.say(#chunks)
httpc:close()
';
}
location = /b {
content_by_lua '
local len = 32768
local t = {}
for i=1,len do
t[i] = 0
end
ngx.print(table.concat(t))
';
}
--- request
GET /a
--- response_body
32768
chunked
3
--- no_error_log
[error]
[warn]
=== TEST 6: Request reader correctly reads body
--- http_config eval: $::HttpConfig
--- config
location = /a {
lua_need_request_body off;
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
local reader, err = httpc:get_client_body_reader(8192)
repeat
local chunk, err = reader()
if chunk then
ngx.print(chunk)
end
until chunk == nil
';
}
--- request
POST /a
foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz
--- response_body: foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz
--- no_error_log
[error]
[warn]
=== TEST 7: Request reader correctly reads body in chunks
--- http_config eval: $::HttpConfig
--- config
location = /a {
lua_need_request_body off;
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
local reader, err = httpc:get_client_body_reader(64)
local chunks = 0
repeat
chunks = chunks +1
local chunk, err = reader()
if chunk then
ngx.print(chunk)
end
until chunk == nil
ngx.say("\\n"..chunks)
';
}
--- request
POST /a
foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz
--- response_body
foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz
3
--- no_error_log
[error]
[warn]
=== TEST 8: Request reader passes into client
--- http_config eval: $::HttpConfig
--- config
location = /a {
lua_need_request_body off;
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local reader, err = httpc:get_client_body_reader(64)
local res, err = httpc:request{
method = POST,
path = "/b",
body = reader,
headers = ngx.req.get_headers(100, true),
}
local body = res:read_body()
ngx.say(body)
httpc:close()
';
}
location = /b {
content_by_lua '
ngx.req.read_body()
local body, err = ngx.req.get_body_data()
ngx.print(body)
';
}
--- request
POST /a
foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz
--- response_body
foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz
--- no_error_log
[error]
[warn]
=== TEST 9: Body reader is a function returning nil when no body is present.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b",
method = "HEAD",
}
repeat
local chunk = res.body_reader()
until not chunk
httpc:close()
';
}
location = /b {
content_by_lua '
ngx.exit(200)
';
}
--- request
GET /a
--- no_error_log
[error]
[warn]
=== TEST 10: Issue a notice (but do not error) if trying to read the request body in a subrequest
--- http_config eval: $::HttpConfig
--- config
location = /a {
echo_location /b;
}
location = /b {
lua_need_request_body off;
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
local reader, err = httpc:get_client_body_reader(8192)
if not reader then
ngx.log(ngx.NOTICE, err)
return
end
repeat
local chunk, err = reader()
if chunk then
ngx.print(chunk)
end
until chunk == nil
';
}
--- request
POST /a
foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz
--- response_body:
--- no_error_log
[error]
[warn]
--- error_log
attempt to read the request body in a subrequest

View file

@ -0,0 +1,145 @@
# vim:set ft= ts=4 sw=4 et:
use Test::Nginx::Socket;
use Cwd qw(cwd);
plan tests => repeat_each() * (blocks() * 4) + 6;
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
error_log logs/error.log debug;
};
$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
no_long_string();
#no_diff();
run_tests();
__DATA__
=== TEST 1: Simple URI interface
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
local res, err = httpc:request_uri("http://127.0.0.1:"..ngx.var.server_port.."/b?a=1&b=2")
if not res then
ngx.log(ngx.ERR, err)
end
ngx.status = res.status
ngx.header["X-Header-A"] = res.headers["X-Header-A"]
ngx.header["X-Header-B"] = res.headers["X-Header-B"]
ngx.print(res.body)
';
}
location = /b {
content_by_lua '
for k,v in pairs(ngx.req.get_uri_args()) do
ngx.header["X-Header-" .. string.upper(k)] = v
end
ngx.say("OK")
';
}
--- request
GET /a
--- response_headers
X-Header-A: 1
X-Header-B: 2
--- response_body
OK
--- no_error_log
[error]
[warn]
=== TEST 2: Simple URI interface HTTP 1.0
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
local res, err = httpc:request_uri(
"http://127.0.0.1:"..ngx.var.server_port.."/b?a=1&b=2", {
}
)
ngx.status = res.status
ngx.header["X-Header-A"] = res.headers["X-Header-A"]
ngx.header["X-Header-B"] = res.headers["X-Header-B"]
ngx.print(res.body)
';
}
location = /b {
content_by_lua '
for k,v in pairs(ngx.req.get_uri_args()) do
ngx.header["X-Header-" .. string.upper(k)] = v
end
ngx.say("OK")
';
}
--- request
GET /a
--- response_headers
X-Header-A: 1
X-Header-B: 2
--- response_body
OK
--- no_error_log
[error]
[warn]
=== TEST 3 Simple URI interface, params override
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
local res, err = httpc:request_uri(
"http://127.0.0.1:"..ngx.var.server_port.."/b?a=1&b=2", {
path = "/c",
query = {
a = 2,
b = 3,
},
}
)
ngx.status = res.status
ngx.header["X-Header-A"] = res.headers["X-Header-A"]
ngx.header["X-Header-B"] = res.headers["X-Header-B"]
ngx.print(res.body)
';
}
location = /c {
content_by_lua '
for k,v in pairs(ngx.req.get_uri_args()) do
ngx.header["X-Header-" .. string.upper(k)] = v
end
ngx.say("OK")
';
}
--- request
GET /a
--- response_headers
X-Header-A: 2
X-Header-B: 3
--- response_body
OK
--- no_error_log
[error]
[warn]

View file

@ -0,0 +1,182 @@
# vim:set ft= ts=4 sw=4 et:
use Test::Nginx::Socket;
use Cwd qw(cwd);
plan tests => repeat_each() * (blocks() * 4);
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
error_log logs/error.log debug;
};
$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
no_long_string();
#no_diff();
run_tests();
__DATA__
=== TEST 1 Simple interface, Connection: Keep-alive. Test the connection is reused.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
local res, err = httpc:request_uri(
"http://127.0.0.1:"..ngx.var.server_port.."/b", {
}
)
ngx.say(res.headers["Connection"])
httpc:connect("127.0.0.1", ngx.var.server_port)
ngx.say(httpc:get_reused_times())
';
}
location = /b {
content_by_lua '
ngx.say("OK")
';
}
--- request
GET /a
--- response_body
keep-alive
1
--- no_error_log
[error]
[warn]
=== TEST 2 Simple interface, Connection: close, test we don't try to keepalive, but also that subsequent connections can keepalive.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
local res, err = httpc:request_uri(
"http://127.0.0.1:"..ngx.var.server_port.."/b", {
version = 1.0,
headers = {
["Connection"] = "close",
},
}
)
httpc:connect("127.0.0.1", ngx.var.server_port)
ngx.say(httpc:get_reused_times())
httpc:set_keepalive()
httpc:connect("127.0.0.1", ngx.var.server_port)
ngx.say(httpc:get_reused_times())
';
}
location = /b {
content_by_lua '
ngx.say("OK")
';
}
--- request
GET /a
--- response_body
0
1
--- no_error_log
[error]
[warn]
=== TEST 3 Generic interface, Connection: Keep-alive. Test the connection is reused.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b"
}
local body = res:read_body()
ngx.say(res.headers["Connection"])
ngx.say(httpc:set_keepalive())
httpc:connect("127.0.0.1", ngx.var.server_port)
ngx.say(httpc:get_reused_times())
';
}
location = /b {
content_by_lua '
ngx.say("OK")
';
}
--- request
GET /a
--- response_body
keep-alive
1
1
--- no_error_log
[error]
[warn]
=== TEST 4 Generic interface, Connection: Close. Test we don't try to keepalive, but also that subsequent connections can keepalive.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
version = 1.0,
headers = {
["Connection"] = "Close",
},
path = "/b"
}
local body = res:read_body()
ngx.say(res.headers["Connection"])
local r, e = httpc:set_keepalive()
ngx.say(r)
ngx.say(e)
httpc:connect("127.0.0.1", ngx.var.server_port)
ngx.say(httpc:get_reused_times())
httpc:set_keepalive()
httpc:connect("127.0.0.1", ngx.var.server_port)
ngx.say(httpc:get_reused_times())
';
}
location = /b {
content_by_lua '
ngx.say("OK")
';
}
--- request
GET /a
--- response_body
close
2
connection must be closed
0
1
--- no_error_log
[error]
[warn]

View file

@ -0,0 +1,143 @@
# vim:set ft= ts=4 sw=4 et:
use Test::Nginx::Socket;
use Cwd qw(cwd);
plan tests => repeat_each() * (blocks() * 4);
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
error_log logs/error.log debug;
};
$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
no_long_string();
#no_diff();
run_tests();
__DATA__
=== TEST 1 Test that pipelined reqests can be read correctly.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local responses = httpc:request_pipeline{
{
path = "/b",
},
{
path = "/c",
},
{
path = "/d",
}
}
for i,r in ipairs(responses) do
if r.status then
ngx.say(r.status)
ngx.say(r.headers["X-Res"])
ngx.say(r:read_body())
end
end
';
}
location = /b {
content_by_lua '
ngx.status = 200
ngx.header["X-Res"] = "B"
ngx.print("B")
';
}
location = /c {
content_by_lua '
ngx.status = 404
ngx.header["X-Res"] = "C"
ngx.print("C")
';
}
location = /d {
content_by_lua '
ngx.status = 200
ngx.header["X-Res"] = "D"
ngx.print("D")
';
}
--- request
GET /a
--- response_body
200
B
B
404
C
C
200
D
D
--- no_error_log
[error]
[warn]
=== TEST 2: Test we can handle timeouts on reading the pipelined requests.
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
httpc:set_timeout(1)
local responses = httpc:request_pipeline{
{
path = "/b",
},
{
path = "/c",
},
}
for i,r in ipairs(responses) do
if r.status then
ngx.say(r.status)
ngx.say(r.headers["X-Res"])
ngx.say(r:read_body())
end
end
';
}
location = /b {
content_by_lua '
ngx.status = 200
ngx.header["X-Res"] = "B"
ngx.print("B")
';
}
location = /c {
content_by_lua '
ngx.status = 404
ngx.header["X-Res"] = "C"
ngx.sleep(1)
ngx.print("C")
';
}
--- request
GET /a
--- response_body
200
B
B
--- no_error_log
[warn]
--- error_log eval
[qr/timeout/]

View file

@ -0,0 +1,59 @@
# vim:set ft= ts=4 sw=4 et:
use Test::Nginx::Socket;
use Cwd qw(cwd);
plan tests => repeat_each() * (blocks() * 4);
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
error_log logs/error.log debug;
};
$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
no_long_string();
#no_diff();
run_tests();
__DATA__
=== TEST 1: parse_uri returns port 443 for https URIs
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
local parsed = httpc:parse_uri("https://www.google.com/foobar")
ngx.say(parsed[3])
';
}
--- request
GET /a
--- response_body
443
--- no_error_log
[error]
[warn]
=== TEST 2: parse_uri returns port 80 for http URIs
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
local parsed = httpc:parse_uri("http://www.google.com/foobar")
ngx.say(parsed[3])
';
}
--- request
GET /a
--- response_body
80
--- no_error_log
[error]
[warn]

View file

@ -0,0 +1,57 @@
# vim:set ft= ts=4 sw=4 et:
use Test::Nginx::Socket;
use Cwd qw(cwd);
plan tests => repeat_each() * (blocks() * 4);
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
};
$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
no_long_string();
#no_diff();
run_tests();
__DATA__
=== TEST 1: Issue a notice (but do not error) if trying to read the request body in a subrequest
--- http_config eval: $::HttpConfig
--- config
location = /a {
echo_location /b;
}
location = /b {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/c",
headers = {
["Content-Type"] = "application/x-www-form-urlencoded",
}
}
if not res then
ngx.say(err)
end
ngx.print(res:read_body())
httpc:close()
';
}
location /c {
echo "OK";
}
--- request
GET /a
--- response_body
OK
--- no_error_log
[error]
[warn]

View file

@ -0,0 +1,152 @@
# vim:set ft= ts=4 sw=4 et:
use Test::Nginx::Socket;
use Cwd qw(cwd);
plan tests => repeat_each() * (blocks() * 5);
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
error_log logs/error.log debug;
};
$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
no_long_string();
#no_diff();
run_tests();
__DATA__
=== TEST 1: Proxy GET request and response
--- http_config eval: $::HttpConfig
--- config
location = /a_prx {
rewrite ^(.*)_prx$ $1 break;
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
httpc:proxy_response(httpc:proxy_request())
httpc:set_keepalive()
';
}
location = /a {
content_by_lua '
ngx.status = 200
ngx.header["X-Test"] = "foo"
ngx.say("OK")
';
}
--- request
GET /a_prx
--- response_body
OK
--- response_headers
X-Test: foo
--- error_code: 200
--- no_error_log
[error]
[warn]
=== TEST 2: Proxy POST request and response
--- http_config eval: $::HttpConfig
--- config
location = /a_prx {
rewrite ^(.*)_prx$ $1 break;
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
httpc:proxy_response(httpc:proxy_request())
httpc:set_keepalive()
';
}
location = /a {
lua_need_request_body on;
content_by_lua '
ngx.status = 404
ngx.header["X-Test"] = "foo"
local args, err = ngx.req.get_post_args()
ngx.say(args["foo"])
ngx.say(args["hello"])
';
}
--- request
POST /a_prx
foo=bar&hello=world
--- response_body
bar
world
--- response_headers
X-Test: foo
--- error_code: 404
--- no_error_log
[error]
[warn]
=== TEST 3: Proxy multiple headers
--- http_config eval: $::HttpConfig
--- config
location = /a_prx {
rewrite ^(.*)_prx$ $1 break;
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
httpc:proxy_response(httpc:proxy_request())
httpc:set_keepalive()
';
}
location = /a {
content_by_lua '
ngx.status = 200
ngx.header["Set-Cookie"] = { "cookie1", "cookie2" }
ngx.say("OK")
';
}
--- request
GET /a_prx
--- response_body
OK
--- raw_response_headers_like: .*Set-Cookie: cookie1\r\nSet-Cookie: cookie2\r\n
--- error_code: 200
--- no_error_log
[error]
[warn]
=== TEST 4: Proxy still works with spaces in URI
--- http_config eval: $::HttpConfig
--- config
location = "/a_ b_prx" {
rewrite ^(.*)_prx$ $1 break;
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
httpc:proxy_response(httpc:proxy_request())
httpc:set_keepalive()
';
}
location = "/a_ b" {
content_by_lua '
ngx.status = 200
ngx.header["X-Test"] = "foo"
ngx.say("OK")
';
}
--- request
GET /a_%20b_prx
--- response_body
OK
--- response_headers
X-Test: foo
--- error_code: 200
--- no_error_log
[error]
[warn]

View file

@ -0,0 +1,160 @@
# vim:set ft= ts=4 sw=4 et:
use Test::Nginx::Socket;
use Cwd qw(cwd);
plan tests => repeat_each() * (blocks() * 4);
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
error_log logs/error.log debug;
};
$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
no_long_string();
#no_diff();
run_tests();
__DATA__
=== TEST 1: Test header normalisation
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http_headers = require "resty.http_headers"
local headers = http_headers.new()
headers.x_a_header = "a"
headers["x-b-header"] = "b"
headers["X-C-Header"] = "c"
headers["X_d-HEAder"] = "d"
ngx.say(headers["X-A-Header"])
ngx.say(headers.x_b_header)
for k,v in pairs(headers) do
ngx.say(k, ": ", v)
end
';
}
--- request
GET /a
--- response_body
a
b
x-b-header: b
x-a-header: a
X-d-HEAder: d
X-C-Header: c
--- no_error_log
[error]
[warn]
=== TEST 2: Test headers can be accessed in all cases
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b"
}
ngx.status = res.status
ngx.say(res.headers["X-Foo-Header"])
ngx.say(res.headers["x-fOo-heaDeR"])
ngx.say(res.headers.x_foo_header)
httpc:close()
';
}
location = /b {
content_by_lua '
ngx.header["X-Foo-Header"] = "bar"
ngx.say("OK")
';
}
--- request
GET /a
--- response_body
bar
bar
bar
--- no_error_log
[error]
[warn]
=== TEST 3: Test request headers are normalised
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http = require "resty.http"
local httpc = http.new()
httpc:connect("127.0.0.1", ngx.var.server_port)
local res, err = httpc:request{
path = "/b",
headers = {
["uSeR-AgENT"] = "test_user_agent",
x_foo = "bar",
},
}
ngx.status = res.status
ngx.print(res:read_body())
httpc:close()
';
}
location = /b {
content_by_lua '
ngx.say(ngx.req.get_headers()["User-Agent"])
ngx.say(ngx.req.get_headers()["X-Foo"])
';
}
--- request
GET /a
--- response_body
test_user_agent
bar
--- no_error_log
[error]
=== TEST 4: Test that headers remain unique
--- http_config eval: $::HttpConfig
--- config
location = /a {
content_by_lua '
local http_headers = require "resty.http_headers"
local headers = http_headers.new()
headers["x-a-header"] = "a"
headers["X-A-HEAder"] = "b"
for k,v in pairs(headers) do
ngx.log(ngx.DEBUG, k, ": ", v)
ngx.header[k] = v
end
';
}
--- request
GET /a
--- response_headers
x-a-header: b
--- no_error_log
[error]
[warn]
[warn]

View file

@ -0,0 +1,63 @@
#!/usr/bin/env perl
use strict;
use warnings;
sub file_contains ($$);
my $version;
for my $file (map glob, qw{ *.lua lib/*.lua lib/*/*.lua lib/*/*/*.lua lib/*/*/*/*.lua lib/*/*/*/*/*.lua }) {
# Check the sanity of each .lua file
open my $in, $file or
die "ERROR: Can't open $file for reading: $!\n";
my $found_ver;
while (<$in>) {
my ($ver, $skipping);
if (/(?x) (?:_VERSION) \s* = .*? ([\d\.]*\d+) (.*? SKIP)?/) {
my $orig_ver = $ver = $1;
$found_ver = 1;
# $skipping = $2;
$ver =~ s{^(\d+)\.(\d{3})(\d{3})$}{join '.', int($1), int($2), int($3)}e;
warn "$file: $orig_ver ($ver)\n";
} elsif (/(?x) (?:_VERSION) \s* = \s* ([a-zA-Z_]\S*)/) {
warn "$file: $1\n";
$found_ver = 1;
last;
}
if ($ver and $version and !$skipping) {
if ($version ne $ver) {
# die "$file: $ver != $version\n";
}
} elsif ($ver and !$version) {
$version = $ver;
}
}
if (!$found_ver) {
warn "WARNING: No \"_VERSION\" or \"version\" field found in `$file`.\n";
}
close $in;
print "Checking use of Lua global variables in file $file ...\n";
system("luac -p -l $file | grep ETGLOBAL | grep -vE 'require|type|tostring|error|ngx|ndk|jit|setmetatable|getmetatable|string|table|io|os|print|tonumber|math|pcall|xpcall|unpack|pairs|ipairs|assert|module|package|coroutine|[gs]etfenv|next|select|rawset|rawget|debug'");
#file_contains($file, "attempt to write to undeclared variable");
system("grep -H -n -E --color '.{120}' $file");
}
sub file_contains ($$) {
my ($file, $regex) = @_;
open my $in, $file
or die "Cannot open $file fo reading: $!\n";
my $content = do { local $/; <$in> };
close $in;
#print "$content";
return scalar ($content =~ /$regex/);
}
if (-d 't') {
for my $file (map glob, qw{ t/*.t t/*/*.t t/*/*/*.t }) {
system(qq{grep -H -n --color -E '\\--- ?(ONLY|LAST)' $file});
}
}

View file

@ -0,0 +1,10 @@
*.swp
*.swo
*~
go
t/servroot/
reindex
nginx
ctags
tags
a.lua

View file

@ -0,0 +1,18 @@
OPENRESTY_PREFIX=/usr/local/openresty
PREFIX ?= /usr/local
LUA_INCLUDE_DIR ?= $(PREFIX)/include
LUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION)
INSTALL ?= install
.PHONY: all test install
all: ;
install: all
$(INSTALL) -d $(DESTDIR)/$(LUA_LIB_DIR)/resty/
$(INSTALL) lib/resty/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/resty/
test: all
PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$$PATH prove -I../test-nginx/lib -r t

View file

@ -0,0 +1,376 @@
Name
====
lua-resty-lock - Simple shm-based nonblocking lock API
Table of Contents
=================
* [Name](#name)
* [Status](#status)
* [Synopsis](#synopsis)
* [Description](#description)
* [Methods](#methods)
* [new](#new)
* [lock](#lock)
* [unlock](#unlock)
* [For Multiple Lua Light Threads](#for-multiple-lua-light-threads)
* [For Cache Locks](#for-cache-locks)
* [Prerequisites](#prerequisites)
* [Installation](#installation)
* [TODO](#todo)
* [Community](#community)
* [English Mailing List](#english-mailing-list)
* [Chinese Mailing List](#chinese-mailing-list)
* [Bugs and Patches](#bugs-and-patches)
* [Author](#author)
* [Copyright and License](#copyright-and-license)
* [See Also](#see-also)
Status
======
This library is still under early development and is production ready.
Synopsis
========
```lua
# nginx.conf
http {
# you do not need the following line if you are using the
# ngx_openresty bundle:
lua_package_path "/path/to/lua-resty-lock/lib/?.lua;;";
lua_shared_dict my_locks 100k;
server {
...
location = /t {
content_by_lua '
local lock = require "resty.lock"
for i = 1, 2 do
local lock = lock:new("my_locks")
local elapsed, err = lock:lock("my_key")
ngx.say("lock: ", elapsed, ", ", err)
local ok, err = lock:unlock()
if not ok then
ngx.say("failed to unlock: ", err)
end
ngx.say("unlock: ", ok)
end
';
}
}
}
```
Description
===========
This library implements a simple mutex lock in a similar way to ngx_proxy module's [proxy_cache_lock directive](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cache_lock).
Under the hood, this library uses [ngx_lua](https://github.com/chaoslawful/lua-nginx-module) module's shared memory dictionaries. The lock waiting is nonblocking because we use stepwise [ngx.sleep](https://github.com/chaoslawful/lua-nginx-module#ngxsleep) to poll the lock periodically.
[Back to TOC](#table-of-contents)
Methods
=======
To load this library,
1. you need to specify this library's path in ngx_lua's [lua_package_path](https://github.com/chaoslawful/lua-nginx-module#lua_package_path) directive. For example, `lua_package_path "/path/to/lua-resty-lock/lib/?.lua;;";`.
2. you use `require` to load the library into a local Lua variable:
```lua
local lock = require "resty.lock"
```
[Back to TOC](#table-of-contents)
new
---
`syntax: obj = lock:new(dict_name)`
`syntax: obj = lock:new(dict_name, opts)`
Creates a new lock object instance by specifying the shared dictionary name (created by [lua_shared_dict](http://https://github.com/chaoslawful/lua-nginx-module#lua_shared_dict)) and an optional options table `opts`.
The options table accepts the following options:
* `exptime`
Specifies expiration time (in seconds) for the lock entry in the shared memory dictionary. You can specify up to `0.001` seconds. Default to 30 (seconds). Even if the invoker does not call `unlock` or the object holding the lock is not GC'd, the lock will be released after this time. So deadlock won't happen even when the worker process holding the lock crashes.
* `timeout`
Specifies the maximal waiting time (in seconds) for the [lock](#lock) method calls on the current object instance. You can specify up to `0.001` seconds. Default to 5 (seconds). This option value cannot be bigger than `exptime`. This timeout is to prevent a [lock](#lock) method call from waiting forever.
You can specify `0` to make the [lock](#lock) method return immediately without waiting if it cannot acquire the lock right away.
* `step`
Specifies the initial step (in seconds) of sleeping when waiting for the lock. Default to `0.001` (seconds). When the [lock](#lock) method is waiting on a busy lock, it sleeps by steps. The step size is increased by a ratio (specified by the `ratio` option) until reaching the step size limit (specified by the `max_step` option).
* `ratio`
Specifies the step increasing ratio. Default to 2, that is, the step size doubles at each waiting iteration.
* `max_step`
Specifies the maximal step size (i.e., sleep interval, in seconds) allowed. See also the `step` and `ratio` options). Default to 0.5 (seconds).
[Back to TOC](#table-of-contents)
lock
----
`syntax: elapsed, err = obj:lock(key)`
Tries to lock a key across all the Nginx worker processes in the current Nginx server instance. Different keys are different locks.
The length of the key string must not be larger than 65535 bytes.
Returns the waiting time (in seconds) if the lock is successfully acquired. Otherwise returns `nil` and a string describing the error.
The waiting time is not from the wallclock, but rather is from simply adding up all the waiting "steps". A nonzero `elapsed` return value indicates that someone else has just hold this lock. But a zero return value cannot gurantee that no one else has just acquired and released the lock.
When this method is waiting on fetching the lock, no operating system threads will be blocked and the current Lua "light thread" will be automatically yielded behind the scene.
It is strongly recommended to always call the [unlock()](#unlock) method to actively release the lock as soon as possible.
If the [unlock()](#unlock) method is never called after this method call, the lock will get released when
1. the current `resty.lock` object instance is collected automatically by the Lua GC.
2. the `exptime` for the lock entry is reached.
Common errors for this method call is
* "timeout"
: The timeout threshold specified by the `timeout` option of the [new](#new) method is exceeded.
* "locked"
: The current `resty.lock` object instance is already holding a lock (not necessarily of the same key).
Other possible errors are from ngx_lua's shared dictionary API.
[Back to TOC](#table-of-contents)
unlock
------
`syntax: ok, err = obj:unlock()`
Releases the lock held by the current `resty.lock` object instance.
Returns `1` on success. Returns `nil` and a string describing the error otherwise.
If you call `unlock` when no lock is currently held, the error "unlocked" will be returned.
[Back to TOC](#table-of-contents)
For Multiple Lua Light Threads
==============================
It is always a bad idea to share a single `resty.lock` object instance across multiple ngx_lua "light threads" because the object itself is stateful and is vulnerable to race conditions. It is highly recommended to always allocate a separate `resty.lock` object instance for each "light thread" that needs one.
[Back to TOC](#table-of-contents)
For Cache Locks
===============
One common use case for this library is avoid the so-called "dog-pile effect", that is, to limit concurrent backend queries for the same key when a cache miss happens. This usage is similar to the standard ngx_proxy module's [proxy_cache_lock](http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cache_lock) directive.
The basic workflow for a cache lock is as follows:
1. Check the cache for a hit with the key. If a cache miss happens, proceed to step 2.
2. Instantiate a `resty.lock` object, call the [lock](#lock) method on the key, and check the 1st return value, i.e., the lock waiting time. If it is `nil`, handle the error; otherwise proceed to step 3.
3. Check the cache again for a hit. If it is still a miss, proceed to step 4; otherwise release the lock by calling [unlock](#unlock) and then return the cached value.
4. Query the backend (the data source) for the value, put the result into the cache, and then release the lock currently held by calling [unlock](#unlock).
Below is a kinda complete code example that demonstrates the idea.
```lua
local resty_lock = require "resty.lock"
local cache = ngx.shared.my_cache
-- step 1:
local val, err = cache:get(key)
if val then
ngx.say("result: ", val)
return
end
if err then
return fail("failed to get key from shm: ", err)
end
-- cache miss!
-- step 2:
local lock = resty_lock:new("my_locks")
local elapsed, err = lock:lock(key)
if not elapsed then
return fail("failed to acquire the lock: ", err)
end
-- lock successfully acquired!
-- step 3:
-- someone might have already put the value into the cache
-- so we check it here again:
val, err = cache:get(key)
if val then
local ok, err = lock:unlock()
if not ok then
return fail("failed to unlock: ", err)
end
ngx.say("result: ", val)
return
end
--- step 4:
local val = fetch_redis(key)
if not val then
local ok, err = lock:unlock()
if not ok then
return fail("failed to unlock: ", err)
end
-- FIXME: we should handle the backend miss more carefully
-- here, like inserting a stub value into the cache.
ngx.say("no value found")
return
end
-- update the shm cache with the newly fetched value
local ok, err = cache:set(key, val, 1)
if not ok then
local ok, err = lock:unlock()
if not ok then
return fail("failed to unlock: ", err)
end
return fail("failed to update shm cache: ", err)
end
local ok, err = lock:unlock()
if not ok then
return fail("failed to unlock: ", err)
end
ngx.say("result: ", val)
```
Here we assume that we use the ngx_lua shared memory dictionary to cache the Redis query results and we have the following configurations in `nginx.conf`:
```nginx
# you may want to change the dictionary size for your cases.
lua_shared_dict my_cache 10m;
lua_shared_dict my_locks 1m;
```
The `my_cache` dictionary is for the data cache while the `my_locks` dictionary is for `resty.lock` itself.
Several important things to note in the example above:
1. You need to release the lock as soon as possible, even when some other unrelated errors happen.
2. You need to update the cache with the result got from the backend *before* releasing the lock so other threads already waiting on the lock can get cached value when they get the lock afterwards.
3. When the backend returns no value at all, we should handle the case carefully by inserting some stub value into the cache.
[Back to TOC](#table-of-contents)
Prerequisites
=============
* [LuaJIT](http://luajit.org) 2.0+
* [ngx_lua](https://github.com/chaoslawful/lua-nginx-module) 0.8.10+
[Back to TOC](#table-of-contents)
Installation
============
It is recommended to use the latest [ngx_openresty bundle](http://openresty.org) directly where this library
is bundled and enabled by default. At least ngx_openresty 1.4.2.9 is required. And you need to enable LuaJIT when building your ngx_openresty
bundle by passing the `--with-luajit` option to its `./configure` script. No extra Nginx configuration is required.
If you want to use this library with your own Nginx build (with ngx_lua), then you need to
ensure you are using at least ngx_lua 0.8.10. Also, You need to configure
the [lua_package_path](https://github.com/chaoslawful/lua-nginx-module#lua_package_path) directive to
add the path of your lua-resty-lock source tree to ngx_lua's Lua module search path, as in
```nginx
# nginx.conf
http {
lua_package_path "/path/to/lua-resty-lock/lib/?.lua;;";
...
}
```
and then load the library in Lua:
```lua
local lock = require "resty.lock"
```
[Back to TOC](#table-of-contents)
TODO
====
* We should simplify the current implementation when LuaJIT 2.1 gets support for `__gc` metamethod on normal Lua tables. Right now we are using an FFI cdata and a ref/unref memo table to work around this, which is rather ugly and a bit inefficient.
[Back to TOC](#table-of-contents)
Community
=========
[Back to TOC](#table-of-contents)
English Mailing List
--------------------
The [openresty-en](https://groups.google.com/group/openresty-en) mailing list is for English speakers.
[Back to TOC](#table-of-contents)
Chinese Mailing List
--------------------
The [openresty](https://groups.google.com/group/openresty) mailing list is for Chinese speakers.
[Back to TOC](#table-of-contents)
Bugs and Patches
================
Please report bugs or submit patches by
1. creating a ticket on the [GitHub Issue Tracker](http://github.com/openresty/lua-resty-lock/issues),
1. or posting to the [OpenResty community](#community).
[Back to TOC](#table-of-contents)
Author
======
Yichun "agentzh" Zhang (章亦春) <agentzh@gmail.com>, CloudFlare Inc.
[Back to TOC](#table-of-contents)
Copyright and License
=====================
This module is licensed under the BSD license.
Copyright (C) 2013-2014, by Yichun "agentzh" Zhang, CloudFlare Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
[Back to TOC](#table-of-contents)
See Also
========
* the ngx_lua module: https://github.com/chaoslawful/lua-nginx-module
* OpenResty: http://openresty.org
[Back to TOC](#table-of-contents)

View file

@ -0,0 +1,208 @@
-- Copyright (C) Yichun Zhang (agentzh)
local ffi = require "ffi"
local ffi_new = ffi.new
local shared = ngx.shared
local sleep = ngx.sleep
local shdict_mt
local debug = ngx.config.debug
local setmetatable = setmetatable
local getmetatable = getmetatable
local tonumber = tonumber
local _M = { _VERSION = '0.04' }
local mt = { __index = _M }
local FREE_LIST_REF = 0
-- FIXME: we don't need this when we have __gc metamethod support on Lua
-- tables.
local memo = {}
if debug then _M.memo = memo end
local function ref_obj(key)
if key == nil then
return -1
end
local ref = memo[FREE_LIST_REF]
if ref and ref ~= 0 then
memo[FREE_LIST_REF] = memo[ref]
else
ref = #memo + 1
end
memo[ref] = key
-- print("ref key_id returned ", ref)
return ref
end
if debug then _M.ref_obj = ref_obj end
local function unref_obj(ref)
if ref >= 0 then
memo[ref] = memo[FREE_LIST_REF]
memo[FREE_LIST_REF] = ref
end
end
if debug then _M.unref_obj = unref_obj end
local function gc_lock(cdata)
local dict_id = tonumber(cdata.dict_id)
local key_id = tonumber(cdata.key_id)
-- print("key_id: ", key_id, ", key: ", memo[key_id], "dict: ",
-- type(memo[cdata.dict_id]))
if key_id > 0 then
local key = memo[key_id]
unref_obj(key_id)
local dict = memo[dict_id]
-- print("dict.delete type: ", type(dict.delete))
local ok, err = dict:delete(key)
if not ok then
ngx.log(ngx.ERR, 'failed to delete key "', key, '": ', err)
end
cdata.key_id = 0
end
unref_obj(dict_id)
end
local ctype = ffi.metatype("struct { int key_id; int dict_id; }",
{ __gc = gc_lock })
function _M.new(_, dict_name, opts)
local dict = shared[dict_name]
if not dict then
return nil, "dictionary not found"
end
local cdata = ffi_new(ctype)
cdata.key_id = 0
cdata.dict_id = ref_obj(dict)
local timeout, exptime, step, ratio, max_step
if opts then
timeout = opts.timeout
exptime = opts.exptime
step = opts.step
ratio = opts.ratio
max_step = opts.max_step
end
if not exptime then
exptime = 30
end
if timeout and timeout > exptime then
timeout = exptime
end
local self = {
cdata = cdata,
dict = dict,
timeout = timeout or 5,
exptime = exptime,
step = step or 0.001,
ratio = ratio or 2,
max_step = max_step or 0.5,
}
return setmetatable(self, mt)
end
function _M.lock(self, key)
if not key then
return nil, "nil key"
end
local dict = self.dict
local cdata = self.cdata
if cdata.key_id > 0 then
return nil, "locked"
end
local exptime = self.exptime
local ok, err = dict:add(key, true, exptime)
if ok then
cdata.key_id = ref_obj(key)
if not shdict_mt then
shdict_mt = getmetatable(dict)
end
return 0
end
if err ~= "exists" then
return nil, err
end
-- lock held by others
local step = self.step
local ratio = self.ratio
local timeout = self.timeout
local max_step = self.max_step
local elapsed = 0
while timeout > 0 do
if step > timeout then
step = timeout
end
sleep(step)
elapsed = elapsed + step
timeout = timeout - step
local ok, err = dict:add(key, true, exptime)
if ok then
cdata.key_id = ref_obj(key)
if not shdict_mt then
shdict_mt = getmetatable(dict)
end
return elapsed
end
if err ~= "exists" then
return nil, err
end
if timeout <= 0 then
break
end
step = step * ratio
if step <= 0 then
step = 0.001
end
if step > max_step then
step = max_step
end
end
return nil, "timeout"
end
function _M.unlock(self)
local dict = self.dict
local cdata = self.cdata
local key_id = tonumber(cdata.key_id)
if key_id <= 0 then
return nil, "unlocked"
end
local key = memo[key_id]
unref_obj(key_id)
local ok, err = dict:delete(key)
if not ok then
return nil, err
end
cdata.key_id = 0
return 1
end
return _M

View file

@ -0,0 +1,470 @@
# vim:set ft= ts=4 sw=4 et:
use Test::Nginx::Socket::Lua;
use Cwd qw(cwd);
repeat_each(2);
plan tests => repeat_each() * (blocks() * 3);
my $pwd = cwd();
our $HttpConfig = qq{
lua_package_path "$pwd/lib/?.lua;;";
lua_package_cpath "/usr/local/openresty-debug/lualib/?.so;/usr/local/openresty/lualib/?.so;;";
lua_shared_dict cache_locks 100k;
};
$ENV{TEST_NGINX_RESOLVER} = '8.8.8.8';
$ENV{TEST_NGINX_REDIS_PORT} ||= 6379;
no_long_string();
#no_diff();
run_tests();
__DATA__
=== TEST 1: lock is subject to garbage collection
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lock = require "resty.lock"
for i = 1, 2 do
collectgarbage("collect")
local lock = lock:new("cache_locks")
local elapsed, err = lock:lock("foo")
ngx.say("lock: ", elapsed, ", ", err)
end
collectgarbage("collect")
';
}
--- request
GET /t
--- response_body
lock: 0, nil
lock: 0, nil
--- no_error_log
[error]
=== TEST 2: serial lock and unlock
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lock = require "resty.lock"
for i = 1, 2 do
local lock = lock:new("cache_locks")
local elapsed, err = lock:lock("foo")
ngx.say("lock: ", elapsed, ", ", err)
local ok, err = lock:unlock()
if not ok then
ngx.say("failed to unlock: ", err)
end
ngx.say("unlock: ", ok)
end
';
}
--- request
GET /t
--- response_body
lock: 0, nil
unlock: 1
lock: 0, nil
unlock: 1
--- no_error_log
[error]
=== TEST 3: timed out locks
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lock = require "resty.lock"
for i = 1, 2 do
local lock1 = lock:new("cache_locks", { timeout = 0.01 })
local lock2 = lock:new("cache_locks", { timeout = 0.01 })
local elapsed, err = lock1:lock("foo")
ngx.say("lock 1: lock: ", elapsed, ", ", err)
local elapsed, err = lock2:lock("foo")
ngx.say("lock 2: lock: ", elapsed, ", ", err)
local ok, err = lock1:unlock()
ngx.say("lock 1: unlock: ", ok, ", ", err)
local ok, err = lock2:unlock()
ngx.say("lock 2: unlock: ", ok, ", ", err)
end
';
}
--- request
GET /t
--- response_body
lock 1: lock: 0, nil
lock 2: lock: nil, timeout
lock 1: unlock: 1, nil
lock 2: unlock: nil, unlocked
lock 1: lock: 0, nil
lock 2: lock: nil, timeout
lock 1: unlock: 1, nil
lock 2: unlock: nil, unlocked
--- no_error_log
[error]
=== TEST 4: waited locks
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local resty_lock = require "resty.lock"
local key = "blah"
local t, err = ngx.thread.spawn(function ()
local lock = resty_lock:new("cache_locks")
local elapsed, err = lock:lock(key)
ngx.say("sub thread: lock: ", elapsed, " ", err)
ngx.sleep(0.1)
ngx.say("sub thread: unlock: ", lock:unlock(key))
end)
local lock = resty_lock:new("cache_locks")
local elapsed, err = lock:lock(key)
ngx.say("main thread: lock: ", elapsed, " ", err)
ngx.say("main thread: unlock: ", lock:unlock())
';
}
--- request
GET /t
--- response_body_like chop
^sub thread: lock: 0 nil
sub thread: unlock: 1
main thread: lock: 0.12[6-9] nil
main thread: unlock: 1
$
--- no_error_log
[error]
=== TEST 5: waited locks (custom step)
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local resty_lock = require "resty.lock"
local key = "blah"
local t, err = ngx.thread.spawn(function ()
local lock = resty_lock:new("cache_locks")
local elapsed, err = lock:lock(key)
ngx.say("sub thread: lock: ", elapsed, " ", err)
ngx.sleep(0.1)
ngx.say("sub thread: unlock: ", lock:unlock(key))
end)
local lock = resty_lock:new("cache_locks", { step = 0.01 })
local elapsed, err = lock:lock(key)
ngx.say("main thread: lock: ", elapsed, " ", err)
ngx.say("main thread: unlock: ", lock:unlock())
';
}
--- request
GET /t
--- response_body_like chop
^sub thread: lock: 0 nil
sub thread: unlock: 1
main thread: lock: 0.1[4-5]\d* nil
main thread: unlock: 1
$
--- no_error_log
[error]
=== TEST 6: waited locks (custom ratio)
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local resty_lock = require "resty.lock"
local key = "blah"
local t, err = ngx.thread.spawn(function ()
local lock = resty_lock:new("cache_locks")
local elapsed, err = lock:lock(key)
ngx.say("sub thread: lock: ", elapsed, " ", err)
ngx.sleep(0.1)
ngx.say("sub thread: unlock: ", lock:unlock(key))
end)
local lock = resty_lock:new("cache_locks", { ratio = 3 })
local elapsed, err = lock:lock(key)
ngx.say("main thread: lock: ", elapsed, " ", err)
ngx.say("main thread: unlock: ", lock:unlock())
';
}
--- request
GET /t
--- response_body_like chop
^sub thread: lock: 0 nil
sub thread: unlock: 1
main thread: lock: 0.1[2]\d* nil
main thread: unlock: 1
$
--- no_error_log
[error]
=== TEST 7: waited locks (custom max step)
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local resty_lock = require "resty.lock"
local key = "blah"
local t, err = ngx.thread.spawn(function ()
local lock = resty_lock:new("cache_locks")
local elapsed, err = lock:lock(key)
ngx.say("sub thread: lock: ", elapsed, " ", err)
ngx.sleep(0.1)
ngx.say("sub thread: unlock: ", lock:unlock(key))
end)
local lock = resty_lock:new("cache_locks", { max_step = 0.05 })
local elapsed, err = lock:lock(key)
ngx.say("main thread: lock: ", elapsed, " ", err)
ngx.say("main thread: unlock: ", lock:unlock())
';
}
--- request
GET /t
--- response_body_like chop
^sub thread: lock: 0 nil
sub thread: unlock: 1
main thread: lock: 0.11[2-4]\d* nil
main thread: unlock: 1
$
--- no_error_log
[error]
=== TEST 8: lock expired by itself
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local resty_lock = require "resty.lock"
local key = "blah"
local t, err = ngx.thread.spawn(function ()
local lock = resty_lock:new("cache_locks", { exptime = 0.1 })
local elapsed, err = lock:lock(key)
ngx.say("sub thread: lock: ", elapsed, " ", err)
ngx.sleep(0.1)
-- ngx.say("sub thread: unlock: ", lock:unlock(key))
end)
local lock = resty_lock:new("cache_locks", { max_step = 0.05 })
local elapsed, err = lock:lock(key)
ngx.say("main thread: lock: ", elapsed, " ", err)
ngx.say("main thread: unlock: ", lock:unlock())
';
}
--- request
GET /t
--- response_body_like chop
^sub thread: lock: 0 nil
main thread: lock: 0.11[2-4]\d* nil
main thread: unlock: 1
$
--- no_error_log
[error]
=== TEST 9: ref & unref (1 at most)
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lock = require "resty.lock"
local memo = lock.memo
local ref = lock.ref_obj("foo")
ngx.say(#memo)
lock.unref_obj(ref)
ngx.say(#memo)
ref = lock.ref_obj("bar")
ngx.say(#memo)
lock.unref_obj(ref)
ngx.say(#memo)
';
}
--- request
GET /t
--- response_body
1
0
1
0
--- no_error_log
[error]
=== TEST 10: ref & unref (2 at most)
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lock = require "resty.lock"
local memo = lock.memo
for i = 1, 2 do
local refs = {}
refs[1] = lock.ref_obj("foo")
ngx.say(#memo)
refs[2] = lock.ref_obj("bar")
ngx.say(#memo)
lock.unref_obj(refs[1])
ngx.say(#memo)
lock.unref_obj(refs[2])
ngx.say(#memo)
end
';
}
--- request
GET /t
--- response_body
1
2
2
2
2
2
1
1
--- no_error_log
[error]
=== TEST 11: lock on a nil key
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lock = require "resty.lock"
local lock = lock:new("cache_locks")
local elapsed, err = lock:lock(nil)
if elapsed then
ngx.say("lock: ", elapsed, ", ", err)
local ok, err = lock:unlock()
if not ok then
ngx.say("failed to unlock: ", err)
end
else
ngx.say("failed to lock: ", err)
end
';
}
--- request
GET /t
--- response_body
failed to lock: nil key
--- no_error_log
[error]
=== TEST 12: same shdict, multple locks
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lock = require "resty.lock"
local memo = lock.memo
local lock1 = lock:new("cache_locks", { timeout = 0.01 })
for i = 1, 3 do
lock1:lock("lock_key")
lock1:unlock()
collectgarbage("collect")
end
local lock2 = lock:new("cache_locks", { timeout = 0.01 })
local lock3 = lock:new("cache_locks", { timeout = 0.01 })
lock2:lock("lock_key")
lock3:lock("lock_key")
collectgarbage("collect")
ngx.say(#memo)
lock2:unlock()
lock3:unlock()
collectgarbage("collect")
';
}
--- request
GET /t
--- response_body
4
--- no_error_log
[error]
=== TEST 13: timed out locks (0 timeout)
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lock = require "resty.lock"
for i = 1, 2 do
local lock1 = lock:new("cache_locks", { timeout = 0 })
local lock2 = lock:new("cache_locks", { timeout = 0 })
local elapsed, err = lock1:lock("foo")
ngx.say("lock 1: lock: ", elapsed, ", ", err)
local elapsed, err = lock2:lock("foo")
ngx.say("lock 2: lock: ", elapsed, ", ", err)
local ok, err = lock1:unlock()
ngx.say("lock 1: unlock: ", ok, ", ", err)
local ok, err = lock2:unlock()
ngx.say("lock 2: unlock: ", ok, ", ", err)
end
';
}
--- request
GET /t
--- response_body
lock 1: lock: 0, nil
lock 2: lock: nil, timeout
lock 1: unlock: 1, nil
lock 2: unlock: nil, unlocked
lock 1: lock: 0, nil
lock 2: lock: nil, timeout
lock 1: unlock: 1, nil
lock 2: unlock: nil, unlocked
--- no_error_log
[error]

View file

@ -0,0 +1,53 @@
# Valgrind suppression file for LuaJIT 2.0.
{
Optimized string compare
Memcheck:Addr4
fun:lj_str_cmp
}
{
Optimized string compare
Memcheck:Addr1
fun:lj_str_cmp
}
{
Optimized string compare
Memcheck:Addr4
fun:lj_str_new
}
{
Optimized string compare
Memcheck:Addr1
fun:lj_str_new
}
{
Optimized string compare
Memcheck:Cond
fun:lj_str_new
}
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_event_process_init
}
{
<insert_a_suppression_name_here>
Memcheck:Param
epoll_ctl(event)
fun:epoll_ctl
fun:ngx_epoll_add_event
}
{
<insert_a_suppression_name_here>
Memcheck:Cond
fun:index
fun:expand_dynamic_string_token
fun:_dl_map_object
fun:map_doit
fun:_dl_catch_error
fun:do_preload
fun:dl_main
fun:_dl_sysdep_start
fun:_dl_start
}

View file

@ -0,0 +1 @@
*.t linguist-language=Text

View file

@ -0,0 +1,10 @@
*.swp
*.swo
*~
go
t/servroot/
reindex
nginx
ctags
tags
a.lua

View file

@ -0,0 +1,19 @@
OPENRESTY_PREFIX=/usr/local/openresty
PREFIX ?= /usr/local
LUA_INCLUDE_DIR ?= $(PREFIX)/include
LUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION)
INSTALL ?= install
.PHONY: all test install
all: ;
install: all
$(INSTALL) -d $(DESTDIR)/$(LUA_LIB_DIR)/resty/lrucache
$(INSTALL) lib/resty/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/resty/
$(INSTALL) lib/resty/lrucache/*.lua $(DESTDIR)/$(LUA_LIB_DIR)/resty/lrucache/
test: all
PATH=$(OPENRESTY_PREFIX)/nginx/sbin:$$PATH prove -I../test-nginx/lib -r t

View file

@ -0,0 +1,293 @@
Name
====
lua-resty-lrucache - in-Lua LRU Cache based on LuaJIT FFI
Table of Contents
=================
* [Name](#name)
* [Status](#status)
* [Synopsis](#synopsis)
* [Description](#description)
* [Methods](#methods)
* [new](#new)
* [set](#set)
* [get](#get)
* [delete](#delete)
* [Prerequisites](#prerequisites)
* [Installation](#installation)
* [TODO](#todo)
* [Community](#community)
* [English Mailing List](#english-mailing-list)
* [Chinese Mailing List](#chinese-mailing-list)
* [Bugs and Patches](#bugs-and-patches)
* [Author](#author)
* [Copyright and License](#copyright-and-license)
* [See Also](#see-also)
Status
======
This library is still under active development and is considered production ready.
Synopsis
========
```lua
-- file myapp.lua: example "myapp" module
local _M = {}
-- alternatively: local lrucache = require "resty.lrucache.pureffi"
local lrucache = require "resty.lrucache"
-- we need to initialize the cache on the lua module level so that
-- it can be shared by all the requests served by each nginx worker process:
local c = lrucache.new(200) -- allow up to 200 items in the cache
if not c then
return error("failed to create the cache: " .. (err or "unknown"))
end
function _M.go()
c:set("dog", 32)
c:set("cat", 56)
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
c:set("dog", { age = 10 }, 0.1) -- expire in 0.1 sec
c:delete("dog")
end
return _M
```
```nginx
# nginx.conf
http {
lua_package_path "/path/to/lua-resty-lrucache/lib/?.lua;;";
server {
listen 8080;
location = /t {
content_by_lua '
require("myapp").go()
';
}
}
}
```
Description
===========
This library implements a simple LRU cache for [OpenResty](http://openresty.org) and the [ngx_lua](https://github.com/openresty/lua-nginx-module) module.
This cache also supports expiration time.
The LRU cache resides completely in the Lua VM and is subject to Lua GC. So do not expect
it to get shared across the OS process boundary. The upside is that you can cache
arbitrary complex Lua values (like deep nested Lua tables) without the overhead of
serialization (as with `ngx_lua`'s
[shared dictionary API](https://github.com/openresty/lua-nginx-module#lua_shared_dict)).
The downside is that your cache is always limited to the current OS process
(like the current nginx worker process). It does not really make much sense to use this
library in the context of [init_by_lua](https://github.com/openresty/lua-nginx-module#lua_shared_dict)
because the cache will not get shared by any of the worker processes
(unless you just want to "warm up" the cache with predefined items which will get
inherited by the workers via `fork`).
There are two different implementations included in this library, in the form of
two classes: `resty.lrucache` and `resty.lrucache.pureffi`. They share exactly the same API. The only difference is that the latter
is a pure FFI implementation that also implements an FFI-based hash table
for the cache lookup while the former uses native Lua tables for it.
If the cache hit rate is relatively high, you should use the `resty.lrucache` class which is faster than `resty.lrucache.pureffi`.
But if the cache hit rate is relatively low and there can be a *lot* of
variations of keys inserted into and removed from the cache, then you should use the `resty.lrucache.pureffi` instead, because
Lua tables are not good at removing keys frequently by design and you
would see the `resizetab` function call in the LuaJIT runtime being very hot in
[on-CPU flame graphs](https://github.com/openresty/stapxx#lj-lua-stacks) if
you use the `resty.lrucache` class instead of `resty.lrucache.pureffi` in this use case.
[Back to TOC](#table-of-contents)
Methods
=======
To load this library,
1. you need to specify this library's path in ngx_lua's [lua_package_path](https://github.com/openresty/lua-nginx-module#lua_package_path) directive. For example, `lua_package_path "/path/to/lua-resty-lrucache/lib/?.lua;;";`.
2. you use `require` to load the library into a local Lua variable:
```lua
local lrucache = require "resty.lrucache"
```
or
```lua
local lrucache = require "resty.lrucache.pureffi"
```
[Back to TOC](#table-of-contents)
new
---
`syntax: cache, err = lrucache.new(max_items [, load_factor])`
Creates a new cache instance. If failed, returns `nil` and a string describing the error.
The `max_items` argument specifies the maximal number of items held in the cache.
The `load-factor` argument designates the "load factor" of the FFI-based hash-table used internally by `resty.lrucache.pureffi`;
the default value is 0.5 (i.e. 50%); if the load factor is specified, it will be clamped
to the range of `[0.1, 1]` (i.e. if load factor is greater than 1, it will be saturated to
1; likewise, if load-factor is smaller than `0.1`, it will be clamped to `0.1`). This argument is only meaningful for `resty.lrucache.pureffi`.
[Back to TOC](#table-of-contents)
set
---
`syntax: cache:set(key, value, ttl)`
Sets a key with a value and an expiration time.
The `ttl` argument specifies the expiration time period. The time value is in seconds, but you can also specify the fraction number part, like `0.25`. A nil `ttl` argument value means never expired (which is the default).
When the cache is full, the cache will automatically evict the least recently used item.
[Back to TOC](#table-of-contents)
get
---
`syntax: data, stale_data = cache:get(key)`
Fetches a value with the key. If the key does not exist in the cache or has already expired, a `nil` value will be returned.
Starting from `v0.03`, the stale data is also returned as the second return value if available.
[Back to TOC](#table-of-contents)
delete
------
`syntax: cache:delete(key)`
Removes an item specified by the key from the cache.
[Back to TOC](#table-of-contents)
Prerequisites
=============
* [LuaJIT](http://luajit.org) 2.0+
* [ngx_lua](https://github.com/openresty/lua-nginx-module) 0.8.10+
[Back to TOC](#table-of-contents)
Installation
============
It is recommended to use the latest [ngx_openresty bundle](http://openresty.org) directly. At least ngx_openresty 1.4.2.9 is required. And you need to enable LuaJIT when building your ngx_openresty
bundle by passing the `--with-luajit` option to its `./configure` script. No extra Nginx configuration is required.
If you want to use this library with your own Nginx build (with ngx_lua), then you need to
ensure you are using at least ngx_lua 0.8.10.
Also, You need to configure
the [lua_package_path](https://github.com/openresty/lua-nginx-module#lua_package_path) directive to
add the path of your lua-resty-lrucache source tree to ngx_lua's Lua module search path, as in
```nginx
# nginx.conf
http {
lua_package_path "/path/to/lua-resty-lrucache/lib/?.lua;;";
...
}
```
and then load the library in Lua:
```lua
local lrucache = require "resty.lrucache"
```
[Back to TOC](#table-of-contents)
TODO
====
* add new method `get_stale` for fetching already expired items.
* add new method `flush_all` for flushing out everything in the cache.
[Back to TOC](#table-of-contents)
Community
=========
[Back to TOC](#table-of-contents)
English Mailing List
--------------------
The [openresty-en](https://groups.google.com/group/openresty-en) mailing list is for English speakers.
[Back to TOC](#table-of-contents)
Chinese Mailing List
--------------------
The [openresty](https://groups.google.com/group/openresty) mailing list is for Chinese speakers.
[Back to TOC](#table-of-contents)
Bugs and Patches
================
Please report bugs or submit patches by
1. creating a ticket on the [GitHub Issue Tracker](https://github.com/openresty/lua-resty-lrucache/issues),
1. or posting to the [OpenResty community](#community).
[Back to TOC](#table-of-contents)
Author
======
Yichun "agentzh" Zhang (章亦春) <agentzh@gmail.com>, CloudFlare Inc.
Shuxin Yang, CloudFlare Inc.
[Back to TOC](#table-of-contents)
Copyright and License
=====================
This module is licensed under the BSD license.
Copyright (C) 2014-2015, by Yichun "agentzh" Zhang, CloudFlare Inc.
Copyright (C) 2014-2015, by Shuxin Yang, CloudFlare Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
[Back to TOC](#table-of-contents)
See Also
========
* the ngx_lua module: https://github.com/chaoslawful/lua-nginx-module
* OpenResty: http://openresty.org
[Back to TOC](#table-of-contents)

View file

@ -0,0 +1,229 @@
-- Copyright (C) Yichun Zhang (agentzh)
local ffi = require "ffi"
local ffi_new = ffi.new
local ffi_sizeof = ffi.sizeof
local ffi_cast = ffi.cast
local ffi_fill = ffi.fill
local ngx_now = ngx.now
local uintptr_t = ffi.typeof("uintptr_t")
local setmetatable = setmetatable
local tonumber = tonumber
-- queue data types
--
-- this queue is a double-ended queue and the first node
-- is reserved for the queue itself.
-- the implementation is mostly borrowed from nginx's ngx_queue_t data
-- structure.
ffi.cdef[[
typedef struct lrucache_queue_s lrucache_queue_t;
struct lrucache_queue_s {
double expire; /* in seconds */
lrucache_queue_t *prev;
lrucache_queue_t *next;
};
]]
local queue_arr_type = ffi.typeof("lrucache_queue_t[?]")
local queue_ptr_type = ffi.typeof("lrucache_queue_t*")
local queue_type = ffi.typeof("lrucache_queue_t")
local NULL = ffi.null
-- queue utility functions
local function queue_insert_tail(h, x)
local last = h[0].prev
x.prev = last
last.next = x
x.next = h
h[0].prev = x
end
local function queue_init(size)
if not size then
size = 0
end
local q = ffi_new(queue_arr_type, size + 1)
ffi_fill(q, ffi_sizeof(queue_type, size + 1), 0)
if size == 0 then
q[0].prev = q
q[0].next = q
else
local prev = q[0]
for i = 1, size do
local e = q[i]
prev.next = e
e.prev = prev
prev = e
end
local last = q[size]
last.next = q
q[0].prev = last
end
return q
end
local function queue_is_empty(q)
-- print("q: ", tostring(q), "q.prev: ", tostring(q), ": ", q == q.prev)
return q == q[0].prev
end
local function queue_remove(x)
local prev = x.prev
local next = x.next
next.prev = prev
prev.next = next
-- for debugging purpose only:
x.prev = NULL
x.next = NULL
end
local function queue_insert_head(h, x)
x.next = h[0].next
x.next.prev = x
x.prev = h
h[0].next = x
end
local function queue_last(h)
return h[0].prev
end
local function queue_head(h)
return h[0].next
end
-- true module stuffs
local _M = {
_VERSION = '0.04'
}
local mt = { __index = _M }
local function ptr2num(ptr)
return tonumber(ffi_cast(uintptr_t, ptr))
end
function _M.new(size)
if size < 1 then
return nil, "size too small"
end
local self = {
keys = {},
hasht = {},
free_queue = queue_init(size),
cache_queue = queue_init(),
key2node = {},
node2key = {},
}
return setmetatable(self, mt)
end
function _M.get(self, key)
local hasht = self.hasht
local val = hasht[key]
if not val then
return nil
end
local node = self.key2node[key]
-- print(key, ": moving node ", tostring(node), " to cache queue head")
local cache_queue = self.cache_queue
queue_remove(node)
queue_insert_head(cache_queue, node)
if node.expire >= 0 and node.expire < ngx_now() then
-- print("expired: ", node.expire, " > ", ngx_now())
return nil, val
end
return val
end
function _M.delete(self, key)
self.hasht[key] = nil
local key2node = self.key2node
local node = key2node[key]
if not node then
return false
end
key2node[key] = nil
self.node2key[ptr2num(node)] = nil
queue_remove(node)
queue_insert_tail(self.free_queue, node)
return true
end
function _M.set(self, key, value, ttl)
local hasht = self.hasht
hasht[key] = value
local key2node = self.key2node
local node = key2node[key]
if not node then
local free_queue = self.free_queue
local node2key = self.node2key
if queue_is_empty(free_queue) then
-- evict the least recently used key
-- assert(not queue_is_empty(self.cache_queue))
node = queue_last(self.cache_queue)
local oldkey = node2key[ptr2num(node)]
-- print(key, ": evicting oldkey: ", oldkey, ", oldnode: ",
-- tostring(node))
if oldkey then
hasht[oldkey] = nil
key2node[oldkey] = nil
end
else
-- take a free queue node
node = queue_head(free_queue)
-- print(key, ": get a new free node: ", tostring(node))
end
node2key[ptr2num(node)] = key
key2node[key] = node
end
queue_remove(node)
queue_insert_head(self.cache_queue, node)
if ttl then
node.expire = ngx_now() + ttl
else
node.expire = -1
end
end
return _M

View file

@ -0,0 +1,534 @@
-- Copyright (C) Yichun Zhang (agentzh)
-- Copyright (C) Shuxin Yang
--[[
This module implements a key/value cache store. We adopt LRU as our
replace/evict policy. Each key/value pair is tagged with a Time-to-Live (TTL);
from user's perspective, stale pairs are automatically removed from the cache.
Why FFI
-------
In Lua, expression "table[key] = nil" does not *PHYSICALLY* remove the value
associated with the key; it just set the value to be nil! So the table will
keep growing with large number of the key/nil pairs which will be purged until
resize() operator is called.
This "feature" is terribly ill-suited to what we need. Therefore we have to
rely on FFI to build a hash-table where any entry can be physically deleted
immediately.
Under the hood:
--------------
In concept, we introduce three data structures to implement the cache store:
1. key/value vector for storing keys and values.
2. a queue to mimic the LRU.
3. hash-table for looking up the value for a given key.
Unfortunately, efficiency and clarity usually come at each other cost. The
data strucutres we are using are slightly more complicated than what we
described above.
o. Lua does not have efficient way to store a vector of pair. So, we use
two vectors for key/value pair: one for keys and the other for values
(_M.key_v and _M.val_v, respectively), and i-th key corresponds to
i-th value.
A key/value pair is identified by the "id" field in a "node" (we shall
discuss node later)
o. The queue is nothing more than a doubly-linked list of "node" linked via
lrucache_pureffi_queue_s::{next|prev} fields.
o. The hash-table has two parts:
- the _M.bucket_v[] a vector of bucket, indiced by hash-value, and
- a bucket is a singly-linked list of "node" via the
lrucache_pureffi_queue_s::conflict field.
A key must be a string, and the hash value of a key is evaluated by:
crc32(key-cast-to-pointer) % size(_M.bucket_v).
We mandate size(_M.bucket_v) being a power-of-two in order to avoid
expensive modulo operation.
At the heart of the module is an array of "node" (of type
lrucache_pureffi_queue_s). A node:
- keeps the meta-data of its corresponding key/value pair
(embodied by the "id", and "expire" field);
- is a part of LRU queue (embodied by "prev" and "next" fields);
- is a part of hash-table (embodied by the "conflict" field).
]]
local ffi = require "ffi"
local bit = require "bit"
local ffi_new = ffi.new
local ffi_sizeof = ffi.sizeof
local ffi_cast = ffi.cast
local ffi_fill = ffi.fill
local ngx_now = ngx.now
local uintptr_t = ffi.typeof("uintptr_t")
local c_str_t = ffi.typeof("const char*")
local int_t = ffi.typeof("int")
local int_array_t = ffi.typeof("int[?]")
local crc_tab = ffi.new("const unsigned int[256]", {
0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F,
0xE963A535, 0x9E6495A3, 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988,
0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, 0x1DB71064, 0x6AB020F2,
0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7,
0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9,
0xFA0F3D63, 0x8D080DF5, 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172,
0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, 0x35B5A8FA, 0x42B2986C,
0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59,
0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423,
0xCFBA9599, 0xB8BDA50F, 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924,
0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, 0x76DC4190, 0x01DB7106,
0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433,
0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D,
0x91646C97, 0xE6635C01, 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E,
0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, 0x65B0D9C6, 0x12B7E950,
0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65,
0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7,
0xA4D1C46D, 0xD3D6F4FB, 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0,
0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, 0x5005713C, 0x270241AA,
0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F,
0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81,
0xB7BD5C3B, 0xC0BA6CAD, 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A,
0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, 0xE3630B12, 0x94643B84,
0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1,
0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB,
0x196C3671, 0x6E6B06E7, 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC,
0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, 0xD6D6A3E8, 0xA1D1937E,
0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B,
0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55,
0x316E8EEF, 0x4669BE79, 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236,
0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, 0xC5BA3BBE, 0xB2BD0B28,
0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D,
0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F,
0x72076785, 0x05005713, 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38,
0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, 0x86D3D2D4, 0xF1D4E242,
0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777,
0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69,
0x616BFFD3, 0x166CCF45, 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2,
0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, 0xAED16A4A, 0xD9D65ADC,
0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9,
0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693,
0x54DE5729, 0x23D967BF, 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94,
0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D });
local setmetatable = setmetatable
local tonumber = tonumber
local tostring = tostring
local type = type
local brshift = bit.rshift
local bxor = bit.bxor
local band = bit.band
local ok, tab_new = pcall(require, "table.new")
if not ok then
tab_new = function (narr, nrec) return {} end
end
-- queue data types
--
-- this queue is a double-ended queue and the first node
-- is reserved for the queue itself.
-- the implementation is mostly borrowed from nginx's ngx_queue_t data
-- structure.
ffi.cdef[[
/* A lrucache_pureffi_queue_s node hook together three data structures:
* o. the key/value store as embodied by the "id" (which is in essence the
* indentifier of key/pair pair) and the "expire" (which is a metadata
* of the corresponding key/pair pair).
* o. The LRU queue via the prev/next fields.
* o. The hash-tabble as embodied by the "conflict" field.
*/
typedef struct lrucache_pureffi_queue_s lrucache_pureffi_queue_t;
struct lrucache_pureffi_queue_s {
/* Each node is assigned a unique ID at construction time, and the
* ID remain immutatble, regardless the node is in active-list or
* free-list. The queue header is assigned ID 0. Since queue-header
* is a sentinel node, 0 denodes "invalid ID".
*
* Intuitively, we can view the "id" as the identifier of key/value
* pair.
*/
int id;
/* The bucket of the hash-table is implemented as a singly-linked list.
* The "conflict" refers to the ID of the next node in the bucket.
*/
int conflict;
double expire; /* in seconds */
lrucache_pureffi_queue_t *prev;
lrucache_pureffi_queue_t *next;
};
]]
local queue_arr_type = ffi.typeof("lrucache_pureffi_queue_t[?]")
--local queue_ptr_type = ffi.typeof("lrucache_pureffi_queue_t*")
local queue_type = ffi.typeof("lrucache_pureffi_queue_t")
local NULL = ffi.null
--========================================================================
--
-- Queue utility functions
--
--========================================================================
-- Append the element "x" to the given queue "h".
local function queue_insert_tail(h, x)
local last = h[0].prev
x.prev = last
last.next = x
x.next = h
h[0].prev = x
end
--[[
Allocate a queue with size + 1 elements. Elements are linked together in a
circular way, i.e. the last element's "next" points to the first element,
while the first element's "prev" element points to the last element.
]]
local function queue_init(size)
if not size then
size = 0
end
local q = ffi_new(queue_arr_type, size + 1)
ffi_fill(q, ffi_sizeof(queue_type, size + 1), 0)
if size == 0 then
q[0].prev = q
q[0].next = q
else
local prev = q[0]
for i = 1, size do
local e = q[i]
e.id = i
prev.next = e
e.prev = prev
prev = e
end
local last = q[size]
last.next = q
q[0].prev = last
end
return q
end
local function queue_is_empty(q)
-- print("q: ", tostring(q), "q.prev: ", tostring(q), ": ", q == q.prev)
return q == q[0].prev
end
local function queue_remove(x)
local prev = x.prev
local next = x.next
next.prev = prev
prev.next = next
-- for debugging purpose only:
x.prev = NULL
x.next = NULL
end
-- Insert the element "x" the to the given queue "h"
local function queue_insert_head(h, x)
x.next = h[0].next
x.next.prev = x
x.prev = h
h[0].next = x
end
local function queue_last(h)
return h[0].prev
end
local function queue_head(h)
return h[0].next
end
--========================================================================
--
-- Miscellaneous Utility Functions
--
--========================================================================
local function ptr2num(ptr)
return tonumber(ffi_cast(uintptr_t, ptr))
end
local function crc32_ptr(ptr)
local p = brshift(ptr2num(ptr), 3)
local b = band(p, 255)
local crc32 = crc_tab[b]
b = band(brshift(p, 8), 255)
crc32 = bxor(brshift(crc32, 8), crc_tab[band(bxor(crc32, b), 255)])
b = band(brshift(p, 16), 255)
crc32 = bxor(brshift(crc32, 8), crc_tab[band(bxor(crc32, b), 255)])
--b = band(brshift(p, 24), 255)
--crc32 = bxor(brshift(crc32, 8), crc_tab[band(bxor(crc32, b), 255)])
return crc32
end
--========================================================================
--
-- Implementation of "export" functions
--
--========================================================================
local _M = {
_VERSION = '0.04'
}
local mt = { __index = _M }
-- "size" specifies the maximum number of entries in the LRU queue, and the
-- "load_factor" designates the 'load factor' of the hash-table we are using
-- internally. The default value of load-factor is 0.5 (i.e. 50%); if the
-- load-factor is specified, it will be clamped to the range of [0.1, 1](i.e.
-- if load-factor is greater than 1, it will be saturated to 1, likewise,
-- if load-factor is smaller than 0.1, it will be clamped to 0.1).
function _M.new(size, load_factor)
if size < 1 then
return nil, "size too small"
end
-- Determine bucket size, which must be power of two.
local load_f = load_factor
if not load_factor then
load_f = 0.5
elseif load_factor > 1 then
load_f = 1
elseif load_factor < 0.1 then
load_f = 0.1
end
local bs_min = size / load_f
-- The bucket_sz *MUST* be a power-of-two. See the hash_string().
local bucket_sz = 1
repeat
bucket_sz = bucket_sz * 2
until bucket_sz >= bs_min
local self = {
size = size,
bucket_sz = bucket_sz,
free_queue = queue_init(size),
cache_queue = queue_init(0),
node_v = nil,
key_v = tab_new(size, 0),
val_v = tab_new(size, 0),
bucket_v = ffi_new(int_array_t, bucket_sz)
}
-- "note_v" is an array of all the nodes used in the LRU queue. Exprpession
-- node_v[i] evaluates to the element of ID "i".
self.node_v = self.free_queue
-- Allocate the array-part of the key_v, val_v, bucket_v.
--local key_v = self.key_v
--local val_v = self.val_v
--local bucket_v = self.bucket_v
ffi_fill(self.bucket_v, ffi_sizeof(int_t, bucket_sz), 0)
return setmetatable(self, mt)
end
local function hash_string(self, str)
local c_str = ffi_cast(c_str_t, str)
local hv = crc32_ptr(c_str)
hv = band(hv, self.bucket_sz - 1)
-- Hint: bucket is 0-based
return hv
end
-- Search the node associated with the key in the bucket, if found returns
-- the the id of the node, and the id of its previous node in the conflict list.
-- The "bucket_hdr_id" is the ID of the first node in the bucket
local function _find_node_in_bucket(key, key_v, node_v, bucket_hdr_id)
if bucket_hdr_id ~= 0 then
local prev = 0
local cur = bucket_hdr_id
while cur ~= 0 and key_v[cur] ~= key do
prev = cur
cur = node_v[cur].conflict
end
if cur ~= 0 then
return cur, prev
end
end
end
-- Return the node corresponding to the key/val.
local function find_key(self, key)
local key_hash = hash_string(self, key)
return _find_node_in_bucket(key, self.key_v, self.node_v,
self.bucket_v[key_hash])
end
--[[ This function tries to
1. Remove the given key and the associated value from the key/value store,
2. Remove the entry associated with the key from the hash-table.
NOTE: all queues remain intact.
If there was a node bound to the key/val, return that node; otherwise,
nil is returned.
]]
local function remove_key(self, key)
local key_v = self.key_v
local val_v = self.val_v
local node_v = self.node_v
local bucket_v = self.bucket_v
local key_hash = hash_string(self, key)
local cur, prev =
_find_node_in_bucket(key, key_v, node_v, bucket_v[key_hash])
if cur then
-- In an attempt to make key and val dead.
key_v[cur] = nil
val_v[cur] = nil
-- Remove the node from the hash table
local next_node = node_v[cur].conflict
if prev ~= 0 then
node_v[prev].conflict = next_node
else
bucket_v[key_hash] = next_node
end
node_v[cur].conflict = 0
return cur
end
end
--[[ Bind the key/val with the given node, and insert the node into the Hashtab.
NOTE: this function does not touch any queue
]]
local function insert_key(self, key, val, node)
-- Bind the key/val with the node
local node_id = node.id
self.key_v[node_id] = key
self.val_v[node_id] = val
-- Insert the node into the hash-table
local key_hash = hash_string(self, key)
local bucket_v = self.bucket_v
node.conflict = bucket_v[key_hash]
bucket_v[key_hash] = node_id
end
function _M.get(self, key)
if type(key) ~= "string" then
key = tostring(key)
end
local node_id = find_key(self, key)
if not node_id then
return nil
end
-- print(key, ": moving node ", tostring(node), " to cache queue head")
local cache_queue = self.cache_queue
local node = self.node_v + node_id
queue_remove(node)
queue_insert_head(cache_queue, node)
local expire = node.expire
if expire >= 0 and expire < ngx_now() then
-- print("expired: ", node.expire, " > ", ngx_now())
return nil, self.val_v[node_id]
end
return self.val_v[node_id]
end
function _M.delete(self, key)
if type(key) ~= "string" then
key = tostring(key)
end
local node_id = remove_key(self, key);
if not node_id then
return false
end
local node = self.node_v + node_id
queue_remove(node)
queue_insert_tail(self.free_queue, node)
return true
end
function _M.set(self, key, value, ttl)
if type(key) ~= "string" then
key = tostring(key)
end
local node_id = find_key(self, key)
local node
if not node_id then
local free_queue = self.free_queue
if queue_is_empty(free_queue) then
-- evict the least recently used key
-- assert(not queue_is_empty(self.cache_queue))
node = queue_last(self.cache_queue)
remove_key(self, self.key_v[node.id])
else
-- take a free queue node
node = queue_head(free_queue)
-- print(key, ": get a new free node: ", tostring(node))
end
-- insert the key
insert_key(self, key, value, node)
else
node = self.node_v + node_id
self.val_v[node_id] = value
end
queue_remove(node)
queue_insert_head(self.cache_queue, node)
if ttl then
node.expire = ngx_now() + ttl
else
node.expire = -1
end
end
return _M

View file

@ -0,0 +1,121 @@
# vim:set ft= ts=4 sw=4 et fdm=marker:
use Test::Nginx::Socket::Lua;
use Cwd qw(cwd);
repeat_each(1);
plan tests => repeat_each() * 13;
#no_diff();
#no_long_string();
my $pwd = cwd();
our $HttpConfig = <<"_EOC_";
lua_package_path "$pwd/lib/?.lua;$pwd/../lua-resty-core/lib/?.lua;;";
#init_by_lua '
#local v = require "jit.v"
#v.on("$Test::Nginx::Util::ErrLogFile")
#require "resty.core"
#';
_EOC_
no_long_string();
run_tests();
__DATA__
=== TEST 1: sanity
--- http_config eval
"$::HttpConfig"
. qq!
init_by_lua '
local function log(...)
ngx.log(ngx.WARN, ...)
end
local lrucache = require "resty.lrucache"
local c = lrucache.new(2)
collectgarbage()
c:set("dog", 32)
c:set("cat", 56)
log("dog: ", c:get("dog"))
log("cat: ", c:get("cat"))
c:set("dog", 32)
c:set("cat", 56)
log("dog: ", c:get("dog"))
log("cat: ", c:get("cat"))
c:delete("dog")
c:delete("cat")
log("dog: ", c:get("dog"))
log("cat: ", c:get("cat"))
';
!
--- config
location = /t {
echo ok;
}
--- request
GET /t
--- response_body
ok
--- no_error_log
[error]
--- error_log
dog: 32
cat: 56
dog: 32
cat: 56
dog: nil
cat: nil
=== TEST 2: sanity
--- http_config eval
"$::HttpConfig"
. qq!
init_by_lua '
lrucache = require "resty.lrucache"
flv_index, err = lrucache.new(200)
if not flv_index then
ngx.log(ngx.ERR, "failed to create the cache: ", err)
return
end
flv_meta, err = lrucache.new(200)
if not flv_meta then
ngx.log(ngx.ERR, "failed to create the cache: ", err)
return
end
flv_channel, err = lrucache.new(200)
if not flv_channel then
ngx.log(ngx.ERR, "failed to create the cache: ", err)
return
end
ngx.log(ngx.WARN, "3 lrucache initialized.")
';
!
--- config
location = /t {
echo ok;
}
--- request
GET /t
--- response_body
ok
--- no_error_log
[error]
--- error_log
3 lrucache initialized.

View file

@ -0,0 +1,75 @@
# vim:set ft= ts=4 sw=4 et fdm=marker:
use Test::Nginx::Socket::Lua;
use Cwd qw(cwd);
repeat_each(2);
plan tests => repeat_each() * (blocks() * 3);
#no_diff();
#no_long_string();
my $pwd = cwd();
our $HttpConfig = <<"_EOC_";
lua_package_path "$pwd/lib/?.lua;$pwd/../lua-resty-core/lib/?.lua;;";
#init_by_lua '
#local v = require "jit.v"
#v.on("$Test::Nginx::Util::ErrLogFile")
#require "resty.core"
#';
_EOC_
no_long_string();
run_tests();
__DATA__
=== TEST 1: sanity
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache"
local c = lrucache.new(2)
collectgarbage()
c:set("dog", 32)
c:set("cat", 56)
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
local lrucache = require "resty.lrucache.pureffi"
local c2 = lrucache.new(2)
ngx.say("dog: ", c2:get("dog"))
ngx.say("cat: ", c2:get("cat"))
c2:set("dog", 9)
c2:set("cat", "hi")
ngx.say("dog: ", c2:get("dog"))
ngx.say("cat: ", c2:get("cat"))
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
';
}
--- request
GET /t
--- response_body
dog: 32
cat: 56
dog: nil
cat: nil
dog: 9
cat: hi
dog: 32
cat: 56
--- no_error_log
[error]

View file

@ -0,0 +1,121 @@
# vim:set ft= ts=4 sw=4 et fdm=marker:
use Test::Nginx::Socket::Lua;
use Cwd qw(cwd);
repeat_each(1);
plan tests => repeat_each() * 13;
#no_diff();
#no_long_string();
my $pwd = cwd();
our $HttpConfig = <<"_EOC_";
lua_package_path "$pwd/lib/?.lua;$pwd/../lua-resty-core/lib/?.lua;;";
#init_by_lua '
#local v = require "jit.v"
#v.on("$Test::Nginx::Util::ErrLogFile")
#require "resty.core"
#';
_EOC_
no_long_string();
run_tests();
__DATA__
=== TEST 1: sanity
--- http_config eval
"$::HttpConfig"
. qq!
init_by_lua '
local function log(...)
ngx.log(ngx.WARN, ...)
end
local lrucache = require "resty.lrucache.pureffi"
local c = lrucache.new(2)
collectgarbage()
c:set("dog", 32)
c:set("cat", 56)
log("dog: ", c:get("dog"))
log("cat: ", c:get("cat"))
c:set("dog", 32)
c:set("cat", 56)
log("dog: ", c:get("dog"))
log("cat: ", c:get("cat"))
c:delete("dog")
c:delete("cat")
log("dog: ", c:get("dog"))
log("cat: ", c:get("cat"))
';
!
--- config
location = /t {
echo ok;
}
--- request
GET /t
--- response_body
ok
--- no_error_log
[error]
--- error_log
dog: 32
cat: 56
dog: 32
cat: 56
dog: nil
cat: nil
=== TEST 2: sanity
--- http_config eval
"$::HttpConfig"
. qq!
init_by_lua '
lrucache = require "resty.lrucache.pureffi"
flv_index, err = lrucache.new(200)
if not flv_index then
ngx.log(ngx.ERR, "failed to create the cache: ", err)
return
end
flv_meta, err = lrucache.new(200)
if not flv_meta then
ngx.log(ngx.ERR, "failed to create the cache: ", err)
return
end
flv_channel, err = lrucache.new(200)
if not flv_channel then
ngx.log(ngx.ERR, "failed to create the cache: ", err)
return
end
ngx.log(ngx.WARN, "3 lrucache initialized.")
';
!
--- config
location = /t {
echo ok;
}
--- request
GET /t
--- response_body
ok
--- no_error_log
[error]
--- error_log
3 lrucache initialized.

View file

@ -0,0 +1,390 @@
# vim:set ft= ts=4 sw=4 et fdm=marker:
use Test::Nginx::Socket::Lua;
use Cwd qw(cwd);
repeat_each(2);
plan tests => repeat_each() * (blocks() * 3);
#no_diff();
#no_long_string();
my $pwd = cwd();
our $HttpConfig = <<"_EOC_";
lua_package_path "$pwd/lib/?.lua;$pwd/../lua-resty-core/lib/?.lua;;";
#init_by_lua '
#local v = require "jit.v"
#v.on("$Test::Nginx::Util::ErrLogFile")
#require "resty.core"
#';
_EOC_
no_long_string();
run_tests();
__DATA__
=== TEST 1: sanity
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache.pureffi"
local c = lrucache.new(2)
collectgarbage()
c:set("dog", 32)
c:set("cat", 56)
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
c:set("dog", 32)
c:set("cat", 56)
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
c:delete("dog")
c:delete("cat")
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
';
}
--- request
GET /t
--- response_body
dog: 32
cat: 56
dog: 32
cat: 56
dog: nil
cat: nil
--- no_error_log
[error]
=== TEST 2: evict existing items
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache.pureffi"
local c = lrucache.new(2)
if not c then
ngx.say("failed to init lrucace: ", err)
return
end
c:set("dog", 32)
c:set("cat", 56)
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
c:set("bird", 76)
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
ngx.say("bird: ", c:get("bird"))
';
}
--- request
GET /t
--- response_body
dog: 32
cat: 56
dog: nil
cat: 56
bird: 76
--- no_error_log
[error]
=== TEST 3: evict existing items (reordered, get should also count)
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache.pureffi"
local c = lrucache.new(2)
if not c then
ngx.say("failed to init lrucace: ", err)
return
end
c:set("cat", 56)
c:set("dog", 32)
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
c:set("bird", 76)
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
ngx.say("bird: ", c:get("bird"))
';
}
--- request
GET /t
--- response_body
dog: 32
cat: 56
dog: nil
cat: 56
bird: 76
--- no_error_log
[error]
=== TEST 4: ttl
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache.pureffi"
local c = lrucache.new(1)
c:set("dog", 32, 0.6)
ngx.say("dog: ", c:get("dog"))
ngx.sleep(0.3)
ngx.say("dog: ", c:get("dog"))
ngx.sleep(0.31)
ngx.say("dog: ", c:get("dog"))
';
}
--- request
GET /t
--- response_body
dog: 32
dog: 32
dog: nil32
--- no_error_log
[error]
=== TEST 5: load factor
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache.pureffi"
local c = lrucache.new(1, 0.25)
ngx.say(c.bucket_sz)
';
}
--- request
GET /t
--- response_body
4
--- no_error_log
[error]
=== TEST 6: load factor clamped to 0.1
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache.pureffi"
local c = lrucache.new(3, 0.05)
ngx.say(c.bucket_sz)
';
}
--- request
GET /t
--- response_body
32
--- no_error_log
[error]
=== TEST 7: load factor saturated to 1
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache.pureffi"
local c = lrucache.new(3, 2.1)
ngx.say(c.bucket_sz)
';
}
--- request
GET /t
--- response_body
4
--- no_error_log
[error]
=== TEST 8: non-string keys
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local function log(...)
ngx.say(...)
end
local lrucache = require "resty.lrucache.pureffi"
local c = lrucache.new(2)
collectgarbage()
local tab1 = {1, 2}
local tab2 = {3, 4}
c:set(tab1, 32)
c:set(tab2, 56)
log("tab1: ", c:get(tab1))
log("tab2: ", c:get(tab2))
c:set(tab1, 32)
c:set(tab2, 56)
log("tab1: ", c:get(tab1))
log("tab2: ", c:get(tab2))
c:delete(tab1)
c:delete(tab2)
log("tab1: ", c:get(tab1))
log("tab2: ", c:get(tab2))
';
}
--- request
GET /t
--- response_body
tab1: 32
tab2: 56
tab1: 32
tab2: 56
tab1: nil
tab2: nil
--- no_error_log
[error]
=== TEST 9: replace value
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache.pureffi"
local c = lrucache.new(1)
c:set("dog", 32)
ngx.say("dog: ", c:get("dog"))
c:set("dog", 33)
ngx.say("dog: ", c:get("dog"))
';
}
--- request
GET /t
--- response_body
dog: 32
dog: 33
--- no_error_log
[error]
=== TEST 10: replace value 2
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache.pureffi"
local c = lrucache.new(1)
c:set("dog", 32, 1.0)
ngx.say("dog: ", c:get("dog"))
c:set("dog", 33, 0.3)
ngx.say("dog: ", c:get("dog"))
ngx.sleep(0.4)
ngx.say("dog: ", c:get("dog"))
';
}
--- request
GET /t
--- response_body
dog: 32
dog: 33
dog: nil33
--- no_error_log
[error]
=== TEST 11: replace value 3 (the old value has longer expire time)
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache.pureffi"
local c = lrucache.new(1)
c:set("dog", 32, 1.2)
c:set("dog", 33, 0.6)
ngx.sleep(0.2)
ngx.say("dog: ", c:get("dog"))
ngx.sleep(0.5)
ngx.say("dog: ", c:get("dog"))
';
}
--- request
GET /t
--- response_body
dog: 33
dog: nil33
--- no_error_log
[error]
=== TEST 12: replace value 4
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache.pureffi"
local c = lrucache.new(1)
c:set("dog", 32, 0.1)
ngx.sleep(0.2)
c:set("dog", 33)
ngx.sleep(0.2)
ngx.say("dog: ", c:get("dog"))
';
}
--- request
GET /t
--- response_body
dog: 33
--- no_error_log
[error]

View file

@ -0,0 +1,250 @@
# vim:set ft= ts=4 sw=4 et fdm=marker:
use Test::Nginx::Socket::Lua;
use Cwd qw(cwd);
repeat_each(2);
plan tests => repeat_each() * (blocks() * 3);
#no_diff();
#no_long_string();
my $pwd = cwd();
our $HttpConfig = <<"_EOC_";
lua_package_path "$pwd/lib/?.lua;$pwd/../lua-resty-core/lib/?.lua;;";
#init_by_lua '
#local v = require "jit.v"
#v.on("$Test::Nginx::Util::ErrLogFile")
#require "resty.core"
#';
_EOC_
no_long_string();
run_tests();
__DATA__
=== TEST 1: sanity
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache"
local c = lrucache.new(2)
collectgarbage()
c:set("dog", 32)
c:set("cat", 56)
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
c:set("dog", 32)
c:set("cat", 56)
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
c:delete("dog")
c:delete("cat")
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
';
}
--- request
GET /t
--- response_body
dog: 32
cat: 56
dog: 32
cat: 56
dog: nil
cat: nil
--- no_error_log
[error]
=== TEST 2: evict existing items
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache"
local c = lrucache.new(2)
if not c then
ngx.say("failed to init lrucace: ", err)
return
end
c:set("dog", 32)
c:set("cat", 56)
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
c:set("bird", 76)
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
ngx.say("bird: ", c:get("bird"))
';
}
--- request
GET /t
--- response_body
dog: 32
cat: 56
dog: nil
cat: 56
bird: 76
--- no_error_log
[error]
=== TEST 3: evict existing items (reordered, get should also count)
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache"
local c = lrucache.new(2)
if not c then
ngx.say("failed to init lrucace: ", err)
return
end
c:set("cat", 56)
c:set("dog", 32)
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
c:set("bird", 76)
ngx.say("dog: ", c:get("dog"))
ngx.say("cat: ", c:get("cat"))
ngx.say("bird: ", c:get("bird"))
';
}
--- request
GET /t
--- response_body
dog: 32
cat: 56
dog: nil
cat: 56
bird: 76
--- no_error_log
[error]
=== TEST 4: ttl
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache"
local c = lrucache.new(1)
c:set("dog", 32, 0.5)
ngx.say("dog: ", c:get("dog"))
ngx.sleep(0.25)
ngx.say("dog: ", c:get("dog"))
ngx.sleep(0.26)
ngx.say("dog: ", c:get("dog"))
';
}
--- request
GET /t
--- response_body
dog: 32
dog: 32
dog: nil32
--- no_error_log
[error]
=== TEST 5: ttl
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache"
local lim = 5
local c = lrucache.new(lim)
local n = 1000
for i = 1, n do
c:set("dog" .. i, i)
c:delete("dog" .. i, i)
c:set("dog" .. i, i)
local cnt = 0
for k, v in pairs(c.hasht) do
cnt = cnt + 1
end
assert(cnt <= lim)
end
for i = 1, n do
local key = "dog" .. math.random(1, n)
c:get(key)
end
for i = 1, n do
local key = "dog" .. math.random(1, n)
c:get(key)
c:set("dog" .. i, i)
local cnt = 0
for k, v in pairs(c.hasht) do
cnt = cnt + 1
end
assert(cnt <= lim)
end
ngx.say("ok")
';
}
--- request
GET /t
--- response_body
ok
--- no_error_log
[error]
--- timeout: 20
=== TEST 6: replace value
--- http_config eval: $::HttpConfig
--- config
location = /t {
content_by_lua '
local lrucache = require "resty.lrucache"
local c = lrucache.new(1)
c:set("dog", 32)
ngx.say("dog: ", c:get("dog"))
c:set("dog", 33)
ngx.say("dog: ", c:get("dog"))
';
}
--- request
GET /t
--- response_body
dog: 32
dog: 33
--- no_error_log
[error]

View file

@ -0,0 +1,36 @@
# Valgrind suppression file for LuaJIT 2.0.
{
<insert_a_suppression_name_here>
Memcheck:Leak
fun:malloc
fun:ngx_alloc
fun:ngx_event_process_init
}
{
<insert_a_suppression_name_here>
Memcheck:Param
epoll_ctl(event)
fun:epoll_ctl
fun:ngx_epoll_add_event
}
{
<insert_a_suppression_name_here>
Memcheck:Cond
fun:index
fun:expand_dynamic_string_token
fun:_dl_map_object
fun:map_doit
fun:_dl_catch_error
fun:do_preload
fun:dl_main
fun:_dl_sysdep_start
fun:_dl_start
}
{
<insert_a_suppression_name_here>
Memcheck:Param
epoll_ctl(event)
fun:epoll_ctl
fun:ngx_epoll_init
fun:ngx_event_process_init
}