diff --git a/README.markdown b/README.markdown index 96c01ebba3..d76236f893 100644 --- a/README.markdown +++ b/README.markdown @@ -4047,6 +4047,7 @@ Nginx API for Lua * [ngx.shared.DICT.expire](#ngxshareddictexpire) * [ngx.shared.DICT.flush_all](#ngxshareddictflush_all) * [ngx.shared.DICT.flush_expired](#ngxshareddictflush_expired) +* [ngx.shared.DICT.flush_expired_tail](#ngxshareddictflush_expired_tail) * [ngx.shared.DICT.get_keys](#ngxshareddictget_keys) * [ngx.shared.DICT.capacity](#ngxshareddictcapacity) * [ngx.shared.DICT.free_space](#ngxshareddictfree_space) @@ -7318,6 +7319,7 @@ The resulting object `dict` has the following methods: * [expire](#ngxshareddictexpire) * [flush_all](#ngxshareddictflush_all) * [flush_expired](#ngxshareddictflush_expired) +* [flush_expired_tail](#ngxshareddictflush_expired_tail) * [get_keys](#ngxshareddictget_keys) * [capacity](#ngxshareddictcapacity) * [free_space](#ngxshareddictfree_space) @@ -7814,6 +7816,25 @@ See also [ngx.shared.DICT.flush_all](#ngxshareddictflush_all) and [ngx.shared.DI [Back to TOC](#nginx-api-for-lua) +ngx.shared.DICT.flush_expired_tail +---------------------------------- + +**syntax:** *flushed = ngx.shared.DICT:flush_expired_tail(max_count)* + +**context:** *init_by_lua*, set_by_lua*, rewrite_by_lua*, access_by_lua*, content_by_lua*, header_filter_by_lua*, body_filter_by_lua*, log_by_lua*, ngx.timer.*, balancer_by_lua*, ssl_certificate_by_lua*, ssl_session_fetch_by_lua*, ssl_session_store_by_lua*, ssl_client_hello_by_lua** + +Scans from the least recently used end of the dictionary's LRU queue, freeing expired items and skipping items without an expiration. Stops at the first item with an expiration time in the future, after freeing `max_count` items, or when the queue ends. Returns the number of items freed. + +The required `max_count` argument must be a positive integer no greater than the maximum C `int` value (normally 2147483647). Zero does not mean unlimited. + +The expiration check and removal are atomic with respect to other shared dictionary operations. This method does not move surviving items in the LRU queue. + +Unlike [flush_expired](#ngxshareddictflush_expired), this method does not scan past an item with an expiration time in the future. Expired items behind such an item may remain indefinitely; applications needing to reclaim those items should also arrange another cleanup mechanism, such as occasional calls to `flush_expired`. + +The limit bounds the number of items freed, not the number inspected or elapsed time. Skipping items without an expiration can require scanning the entire dictionary. Removing an expired list also frees all its elements while holding the dictionary lock; one list counts as one item towards `max_count`. + +[Back to TOC](#nginx-api-for-lua) + ngx.shared.DICT.get_keys ------------------------ diff --git a/src/ngx_http_lua_shdict.c b/src/ngx_http_lua_shdict.c index 9d9f4ff8cf..09676337ef 100644 --- a/src/ngx_http_lua_shdict.c +++ b/src/ngx_http_lua_shdict.c @@ -21,6 +21,9 @@ static ngx_int_t ngx_http_lua_shdict_lookup(ngx_shm_zone_t *shm_zone, ngx_uint_t hash, u_char *kdata, size_t klen, ngx_http_lua_shdict_node_t **sdp); static int ngx_http_lua_shdict_flush_expired(lua_State *L); +static int ngx_http_lua_shdict_flush_expired_tail(lua_State *L); +static int ngx_http_lua_shdict_flush_expired_helper(lua_State *L, + int tail_only); static int ngx_http_lua_shdict_get_keys(lua_State *L); static int ngx_http_lua_shdict_lpush(lua_State *L); static int ngx_http_lua_shdict_rpush(lua_State *L); @@ -317,7 +320,7 @@ ngx_http_lua_inject_shdict_api(ngx_http_lua_main_conf_t *lmcf, lua_State *L) lua_createtable(L, 0, lmcf->shdict_zones->nelts /* nrec */); /* ngx.shared */ - lua_createtable(L, 0 /* narr */, 22 /* nrec */); /* shared mt */ + lua_createtable(L, 0 /* narr */, 23 /* nrec */); /* shared mt */ lua_pushcfunction(L, ngx_http_lua_shdict_lpush); lua_setfield(L, -2, "lpush"); @@ -337,6 +340,9 @@ ngx_http_lua_inject_shdict_api(ngx_http_lua_main_conf_t *lmcf, lua_State *L) lua_pushcfunction(L, ngx_http_lua_shdict_flush_expired); lua_setfield(L, -2, "flush_expired"); + lua_pushcfunction(L, ngx_http_lua_shdict_flush_expired_tail); + lua_setfield(L, -2, "flush_expired_tail"); + lua_pushcfunction(L, ngx_http_lua_shdict_get_keys); lua_setfield(L, -2, "get_keys"); @@ -393,6 +399,20 @@ ngx_http_lua_shdict_get_zone(lua_State *L, int index) static int ngx_http_lua_shdict_flush_expired(lua_State *L) +{ + return ngx_http_lua_shdict_flush_expired_helper(L, 0); +} + + +static int +ngx_http_lua_shdict_flush_expired_tail(lua_State *L) +{ + return ngx_http_lua_shdict_flush_expired_helper(L, 1); +} + + +static int +ngx_http_lua_shdict_flush_expired_helper(lua_State *L, int tail_only) { ngx_queue_t *q, *prev, *list_queue, *lq; ngx_http_lua_shdict_node_t *sd; @@ -404,10 +424,15 @@ ngx_http_lua_shdict_flush_expired(lua_State *L) ngx_rbtree_node_t *node; uint64_t now; int n; + lua_Number limit; ngx_http_lua_shdict_list_node_t *lnode; n = lua_gettop(L); + if (tail_only && n != 2) { + return luaL_error(L, "expecting 2 arguments, but saw %d", n); + } + if (n != 1 && n != 2) { return luaL_error(L, "expecting 1 or 2 argument(s), but saw %d", n); } @@ -419,7 +444,16 @@ ngx_http_lua_shdict_flush_expired(lua_State *L) return luaL_error(L, "bad user data for the ngx_shm_zone_t pointer"); } - if (n == 2) { + if (tail_only) { + limit = luaL_checknumber(L, 2); + if (!(limit >= 1 && limit <= INT_MAX) || limit != (int) limit) { + return luaL_argerror(L, 2, + "positive integer within int range expected"); + } + + attempts = (int) limit; + + } else if (n == 2) { attempts = luaL_checkint(L, 2); } @@ -472,6 +506,9 @@ ngx_http_lua_shdict_flush_expired(lua_State *L) if (attempts && freed == attempts) { break; } + + } else if (tail_only && sd->expires != 0) { + break; } q = prev; diff --git a/t/195-shdict-flush-expired-tail.t b/t/195-shdict-flush-expired-tail.t new file mode 100644 index 0000000000..9082df3a4e --- /dev/null +++ b/t/195-shdict-flush-expired-tail.t @@ -0,0 +1,307 @@ +# vim:set ft= ts=4 sw=4 et fdm=marker: +use Test::Nginx::Socket::Lua; + +repeat_each(2); +plan tests => repeat_each() * blocks() * 3; +no_long_string(); +run_tests(); + +__DATA__ + +=== TEST 1: empty dictionary +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local d = ngx.shared.dogs + ngx.say(d:flush_expired_tail(1)) + } + } +--- request +GET /test +--- response_body +0 +--- no_error_log +[error] + +=== TEST 2: limit and continuation +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local d = ngx.shared.dogs + for i = 1, 3 do assert(d:set(i, i, 0.01)) end + ngx.sleep(0.03) + ngx.say(d:flush_expired_tail(2)) + ngx.say(d:get_stale(1)) + ngx.say(d:get_stale(2)) + ngx.say(d:get_stale(3)) + ngx.say(d:flush_expired_tail(2)) + ngx.say(d:flush_expired_tail(2)) + } + } +--- request +GET /test +--- response_body +2 +nil +nil +3niltrue +1 +0 +--- no_error_log +[error] + +=== TEST 3: permanent tail is skipped +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local d = ngx.shared.dogs + assert(d:set("blocker", 1)) + assert(d:set("expired", 2, 0.01)) + ngx.sleep(0.03) + ngx.say(d:flush_expired_tail(10)) + ngx.say(d:flush_expired_tail(10)) + ngx.say(d:flush_expired()) + ngx.say(d:get("blocker")) + } + } +--- request +GET /test +--- response_body +1 +0 +0 +1 +--- no_error_log +[error] + +=== TEST 4: stop after expired prefix at live item +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local d = ngx.shared.dogs + assert(d:set("old", 1, 0.01)) + assert(d:set("live", 2, 60)) + assert(d:set("new", 3, 0.01)) + ngx.sleep(0.03) + ngx.say(d:flush_expired_tail(10)) + ngx.say(d:get_stale("old")) + ngx.say(d:get_stale("new")) + ngx.say(d:flush_expired(10)) + } + } +--- request +GET /test +--- response_body +1 +nil +3niltrue +1 +--- no_error_log +[error] + +=== TEST 5: successful access moves blocker away from tail +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local d = ngx.shared.dogs + assert(d:set("blocker", 1, 60)) + assert(d:set("expired", 2, 0.01)) + assert(d:get("blocker") == 1) + ngx.sleep(0.03) + ngx.say(d:flush_expired_tail(10)) + ngx.say(d:get("blocker")) + } + } +--- request +GET /test +--- response_body +1 +1 +--- no_error_log +[error] + +=== TEST 6: renewal before cleanup preserves node +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local d = ngx.shared.dogs + assert(d:set("renewed", 1, 0.01)) + assert(d:set("expired", 2, 0.01)) + ngx.sleep(0.03) + assert(d:expire("renewed", 60)) + ngx.say(d:flush_expired_tail(10)) + ngx.say(d:get("renewed")) + ngx.say(d:flush_expired()) + } + } +--- request +GET /test +--- response_body +0 +1 +1 +--- no_error_log +[error] + +=== TEST 7: expired list frees members and counts once +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local d = ngx.shared.dogs + local before = d:free_space() + for i = 1, 100 do assert(d:rpush("list", string.rep("x", 1024))) end + assert(d:expire("list", 0.01)) + assert(d:set("next", 2, 0.01)) + ngx.sleep(0.03) + ngx.say(d:flush_expired_tail(1)) + ngx.say(d:get_stale("list")) + ngx.say(d:flush_expired_tail(1)) + ngx.say(d:free_space() == before) + } + } +--- request +GET /test +--- response_body +1 +nil +1 +true +--- no_error_log +[error] + +=== TEST 8: surviving LRU order is unchanged +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local d = ngx.shared.dogs + assert(d:set("first", 1, 60)) + assert(d:set("second", 2)) + assert(d:expire("second", 0.01)) + ngx.say(d:flush_expired_tail(10)) + ngx.sleep(0.03) + ngx.say(d:flush_expired_tail(10)) + ngx.say(d:flush_expired()) + } + } +--- request +GET /test +--- response_body +0 +0 +1 +--- no_error_log +[error] + +=== TEST 9: invalid limits +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local d = ngx.shared.dogs + for _, limit in ipairs({0, -1, 0.5, 1.5, 2147483648, math.huge, + -math.huge, 0/0, false, {}, "bad"}) do + local ok = pcall(d.flush_expired_tail, d, limit) + ngx.say(ok) + end + ngx.say((pcall(d.flush_expired_tail, d))) + ngx.say((pcall(d.flush_expired_tail, d, 1, 2))) + ngx.say(d:flush_expired_tail(2147483647)) + } + } +--- request +GET /test +--- response_body +false +false +false +false +false +false +false +false +false +false +false +false +false +0 +--- no_error_log +[error] + +=== TEST 10: original flush still scans past live items and honors its limit +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local d = ngx.shared.dogs + assert(d:set("blocker", 1, 60)) + for i = 1, 3 do assert(d:set(i, i, 0.01)) end + ngx.sleep(0.03) + ngx.say(d:flush_expired_tail(10)) + ngx.say(d:flush_expired(1)) + ngx.say(d:flush_expired(0)) + ngx.say(d:flush_expired()) + ngx.say(d:get("blocker")) + } + } +--- request +GET /test +--- response_body +0 +1 +2 +0 +1 +--- no_error_log +[error] + +=== TEST 11: permanent items between expired items do not consume the limit +--- http_config + lua_shared_dict dogs 1m; +--- config + location = /test { + content_by_lua_block { + local d = ngx.shared.dogs + assert(d:set("permanent1", 1)) + assert(d:set("expired1", 2, 0.01)) + assert(d:set("permanent2", 3)) + assert(d:set("expired2", 4, 0.01)) + assert(d:set("expired3", 5, 0.01)) + ngx.sleep(0.03) + ngx.say(d:flush_expired_tail(2)) + ngx.say(d:get_stale("expired3")) + ngx.say(d:flush_expired_tail(2)) + ngx.say(d:get("permanent1")) + ngx.say(d:get("permanent2")) + ngx.say(d:flush_expired_tail(2)) + } + } +--- request +GET /test +--- response_body +2 +5niltrue +1 +1 +3 +0 +--- no_error_log +[error]