diff --git a/lib/docs/filters/redis/clean_html.rb b/lib/docs/filters/redis/clean_html.rb
index 5b3bc5e4..8eb3a428 100644
--- a/lib/docs/filters/redis/clean_html.rb
+++ b/lib/docs/filters/redis/clean_html.rb
@@ -3,14 +3,20 @@ module Docs
class CleanHtmlFilter < Filter
def call
if root_page?
- at_css('ul')['class'] = 'commands'
+ if root_page?
+ slug_types = {}
+ css('article[data-group]').each do |node|
+ slug_types[node.at_css('a')['href']] = node['data-group']
+ end
+ # binding.pry
+ end
else
title = at_css('h1')
title.after("#{title.content.strip}")
title.content = title.content.split(' ').first
end
- css('nav', 'aside', 'form', '.anchor-link').remove
+ css('nav', 'aside', '.page-feedback', '.anchor-link').remove
css('> article', '.article-main', 'pre > code', '.container').each do |node|
node.before(node.children).remove
diff --git a/lib/docs/filters/redis/entries.rb b/lib/docs/filters/redis/entries.rb
index c0dc2fa4..f1073d69 100644
--- a/lib/docs/filters/redis/entries.rb
+++ b/lib/docs/filters/redis/entries.rb
@@ -1,30 +1,510 @@
module Docs
class Redis
class EntriesFilter < Docs::EntriesFilter
+
def get_name
- slug.gsub('-', ' ')
+ slug.gsub('-', ' ').remove('/')
end
def get_type
- case at_css('aside > ul:last-child a').content.strip
- when 'DEL' then 'Keys'
- when 'APPEND' then 'Strings'
- when 'HDEL' then 'Hashes'
- when 'BLPOP' then 'Lists'
- when 'SADD' then 'Sets'
- when 'BZPOPMAX' then 'Sorted Sets'
- when 'PSUBSCRIBE' then 'Pub/Sub'
- when 'DISCARD' then 'Transactions'
- when 'EVAL' then 'Scripting'
- when 'AUTH' then 'Connection'
- when 'BGREWRITEAOF' then 'Server'
- when 'PFADD' then 'HyperLogLog'
- when 'CLUSTER ADDSLOTS' then 'Cluster'
- when 'GEOADD' then 'Geo'
- when 'XACK' then 'Stream'
- else 'Miscellaneous'
- end
+ SLUG_NAMES[SLUG_TYPES[slug]]
end
+
+ # obtain from https://redis.io/commands/ via lib/docs/filters/redis/clean_html.rb using `binding.pry`
+ SLUG_TYPES = {
+ "acl-cat/" => "server",
+ "acl-deluser/" => "server",
+ "acl-dryrun/" => "server",
+ "acl-genpass/" => "server",
+ "acl-getuser/" => "server",
+ "acl-list/" => "server",
+ "acl-load/" => "server",
+ "acl-log/" => "server",
+ "acl-save/" => "server",
+ "acl-setuser/" => "server",
+ "acl-users/" => "server",
+ "acl-whoami/" => "server",
+ "append/" => "string",
+ "asking/" => "cluster",
+ "auth/" => "connection",
+ "bf.add/" => "bf",
+ "bf.card/" => "bf",
+ "bf.exists/" => "bf",
+ "bf.info/" => "bf",
+ "bf.insert/" => "bf",
+ "bf.loadchunk/" => "bf",
+ "bf.madd/" => "bf",
+ "bf.mexists/" => "bf",
+ "bf.reserve/" => "bf",
+ "bf.scandump/" => "bf",
+ "bgrewriteaof/" => "server",
+ "bgsave/" => "server",
+ "bitcount/" => "bitmap",
+ "bitfield/" => "bitmap",
+ "bitfield_ro/" => "bitmap",
+ "bitop/" => "bitmap",
+ "bitpos/" => "bitmap",
+ "blmove/" => "list",
+ "blmpop/" => "list",
+ "blpop/" => "list",
+ "brpop/" => "list",
+ "brpoplpush/" => "list",
+ "bzmpop/" => "sorted-set",
+ "bzpopmax/" => "sorted-set",
+ "bzpopmin/" => "sorted-set",
+ "cf.add/" => "cf",
+ "cf.addnx/" => "cf",
+ "cf.count/" => "cf",
+ "cf.del/" => "cf",
+ "cf.exists/" => "cf",
+ "cf.info/" => "cf",
+ "cf.insert/" => "cf",
+ "cf.insertnx/" => "cf",
+ "cf.loadchunk/" => "cf",
+ "cf.mexists/" => "cf",
+ "cf.reserve/" => "cf",
+ "cf.scandump/" => "cf",
+ "client-caching/" => "connection",
+ "client-getname/" => "connection",
+ "client-getredir/" => "connection",
+ "client-id/" => "connection",
+ "client-info/" => "connection",
+ "client-kill/" => "connection",
+ "client-list/" => "connection",
+ "client-no-evict/" => "connection",
+ "client-pause/" => "connection",
+ "client-reply/" => "connection",
+ "client-setname/" => "connection",
+ "client-tracking/" => "connection",
+ "client-trackinginfo/" => "connection",
+ "client-unblock/" => "connection",
+ "client-unpause/" => "connection",
+ "cluster-addslots/" => "cluster",
+ "cluster-addslotsrange/" => "cluster",
+ "cluster-bumpepoch/" => "cluster",
+ "cluster-count-failure-reports/" => "cluster",
+ "cluster-countkeysinslot/" => "cluster",
+ "cluster-delslots/" => "cluster",
+ "cluster-delslotsrange/" => "cluster",
+ "cluster-failover/" => "cluster",
+ "cluster-flushslots/" => "cluster",
+ "cluster-forget/" => "cluster",
+ "cluster-getkeysinslot/" => "cluster",
+ "cluster-info/" => "cluster",
+ "cluster-keyslot/" => "cluster",
+ "cluster-links/" => "cluster",
+ "cluster-meet/" => "cluster",
+ "cluster-myid/" => "cluster",
+ "cluster-nodes/" => "cluster",
+ "cluster-replicas/" => "cluster",
+ "cluster-replicate/" => "cluster",
+ "cluster-reset/" => "cluster",
+ "cluster-saveconfig/" => "cluster",
+ "cluster-set-config-epoch/" => "cluster",
+ "cluster-setslot/" => "cluster",
+ "cluster-shards/" => "cluster",
+ "cluster-slaves/" => "cluster",
+ "cluster-slots/" => "cluster",
+ "cms.incrby/" => "cms",
+ "cms.info/" => "cms",
+ "cms.initbydim/" => "cms",
+ "cms.initbyprob/" => "cms",
+ "cms.merge/" => "cms",
+ "cms.query/" => "cms",
+ "command/" => "server",
+ "command-count/" => "server",
+ "command-docs/" => "server",
+ "command-getkeys/" => "server",
+ "command-getkeysandflags/" => "server",
+ "command-info/" => "server",
+ "command-list/" => "server",
+ "config-get/" => "server",
+ "config-resetstat/" => "server",
+ "config-rewrite/" => "server",
+ "config-set/" => "server",
+ "copy/" => "generic",
+ "dbsize/" => "server",
+ "decr/" => "string",
+ "decrby/" => "string",
+ "del/" => "generic",
+ "discard/" => "transactions",
+ "dump/" => "generic",
+ "echo/" => "connection",
+ "eval/" => "scripting",
+ "eval_ro/" => "scripting",
+ "evalsha/" => "scripting",
+ "evalsha_ro/" => "scripting",
+ "exec/" => "transactions",
+ "exists/" => "generic",
+ "expire/" => "generic",
+ "expireat/" => "generic",
+ "expiretime/" => "generic",
+ "failover/" => "server",
+ "fcall/" => "scripting",
+ "fcall_ro/" => "scripting",
+ "flushall/" => "server",
+ "flushdb/" => "server",
+ "ft._list/" => "search",
+ "ft.aggregate/" => "search",
+ "ft.aliasadd/" => "search",
+ "ft.aliasdel/" => "search",
+ "ft.aliasupdate/" => "search",
+ "ft.alter/" => "search",
+ "ft.config-get/" => "search",
+ "ft.config-set/" => "search",
+ "ft.create/" => "search",
+ "ft.cursor-del/" => "search",
+ "ft.cursor-read/" => "search",
+ "ft.dictadd/" => "search",
+ "ft.dictdel/" => "search",
+ "ft.dictdump/" => "search",
+ "ft.dropindex/" => "search",
+ "ft.explain/" => "search",
+ "ft.explaincli/" => "search",
+ "ft.info/" => "search",
+ "ft.profile/" => "search",
+ "ft.search/" => "search",
+ "ft.spellcheck/" => "search",
+ "ft.sugadd/" => "suggestion",
+ "ft.sugdel/" => "suggestion",
+ "ft.sugget/" => "suggestion",
+ "ft.suglen/" => "suggestion",
+ "ft.syndump/" => "search",
+ "ft.synupdate/" => "search",
+ "ft.tagvals/" => "search",
+ "function-delete/" => "scripting",
+ "function-dump/" => "scripting",
+ "function-flush/" => "scripting",
+ "function-kill/" => "scripting",
+ "function-list/" => "scripting",
+ "function-load/" => "scripting",
+ "function-restore/" => "scripting",
+ "function-stats/" => "scripting",
+ "geoadd/" => "geo",
+ "geodist/" => "geo",
+ "geohash/" => "geo",
+ "geopos/" => "geo",
+ "georadius/" => "geo",
+ "georadius_ro/" => "geo",
+ "georadiusbymember/" => "geo",
+ "georadiusbymember_ro/" => "geo",
+ "geosearch/" => "geo",
+ "geosearchstore/" => "geo",
+ "get/" => "string",
+ "getbit/" => "bitmap",
+ "getdel/" => "string",
+ "getex/" => "string",
+ "getrange/" => "string",
+ "getset/" => "string",
+ "graph.config-get/" => "graph",
+ "graph.config-set/" => "graph",
+ "graph.delete/" => "graph",
+ "graph.explain/" => "graph",
+ "graph.list/" => "graph",
+ "graph.profile/" => "graph",
+ "graph.query/" => "graph",
+ "graph.ro_query/" => "graph",
+ "graph.slowlog/" => "graph",
+ "hdel/" => "hash",
+ "hello/" => "connection",
+ "hexists/" => "hash",
+ "hget/" => "hash",
+ "hgetall/" => "hash",
+ "hincrby/" => "hash",
+ "hincrbyfloat/" => "hash",
+ "hkeys/" => "hash",
+ "hlen/" => "hash",
+ "hmget/" => "hash",
+ "hmset/" => "hash",
+ "hrandfield/" => "hash",
+ "hscan/" => "hash",
+ "hset/" => "hash",
+ "hsetnx/" => "hash",
+ "hstrlen/" => "hash",
+ "hvals/" => "hash",
+ "incr/" => "string",
+ "incrby/" => "string",
+ "incrbyfloat/" => "string",
+ "info/" => "server",
+ "json.arrappend/" => "json",
+ "json.arrindex/" => "json",
+ "json.arrinsert/" => "json",
+ "json.arrlen/" => "json",
+ "json.arrpop/" => "json",
+ "json.arrtrim/" => "json",
+ "json.clear/" => "json",
+ "json.debug/" => "json",
+ "json.debug-memory/" => "json",
+ "json.del/" => "json",
+ "json.forget/" => "json",
+ "json.get/" => "json",
+ "json.mget/" => "json",
+ "json.numincrby/" => "json",
+ "json.nummultby/" => "json",
+ "json.objkeys/" => "json",
+ "json.objlen/" => "json",
+ "json.resp/" => "json",
+ "json.set/" => "json",
+ "json.strappend/" => "json",
+ "json.strlen/" => "json",
+ "json.toggle/" => "json",
+ "json.type/" => "json",
+ "keys/" => "generic",
+ "lastsave/" => "server",
+ "latency-doctor/" => "server",
+ "latency-graph/" => "server",
+ "latency-histogram/" => "server",
+ "latency-history/" => "server",
+ "latency-latest/" => "server",
+ "latency-reset/" => "server",
+ "lcs/" => "string",
+ "lindex/" => "list",
+ "linsert/" => "list",
+ "llen/" => "list",
+ "lmove/" => "list",
+ "lmpop/" => "list",
+ "lolwut/" => "server",
+ "lpop/" => "list",
+ "lpos/" => "list",
+ "lpush/" => "list",
+ "lpushx/" => "list",
+ "lrange/" => "list",
+ "lrem/" => "list",
+ "lset/" => "list",
+ "ltrim/" => "list",
+ "memory-doctor/" => "server",
+ "memory-malloc-stats/" => "server",
+ "memory-purge/" => "server",
+ "memory-stats/" => "server",
+ "memory-usage/" => "server",
+ "mget/" => "string",
+ "migrate/" => "generic",
+ "module-list/" => "server",
+ "module-load/" => "server",
+ "module-loadex/" => "server",
+ "module-unload/" => "server",
+ "monitor/" => "server",
+ "move/" => "generic",
+ "mset/" => "string",
+ "msetnx/" => "string",
+ "multi/" => "transactions",
+ "object-encoding/" => "generic",
+ "object-freq/" => "generic",
+ "object-idletime/" => "generic",
+ "object-refcount/" => "generic",
+ "persist/" => "generic",
+ "pexpire/" => "generic",
+ "pexpireat/" => "generic",
+ "pexpiretime/" => "generic",
+ "pfadd/" => "hyperloglog",
+ "pfcount/" => "hyperloglog",
+ "pfdebug/" => "hyperloglog",
+ "pfmerge/" => "hyperloglog",
+ "pfselftest/" => "hyperloglog",
+ "ping/" => "connection",
+ "psetex/" => "string",
+ "psubscribe/" => "pubsub",
+ "psync/" => "server",
+ "pttl/" => "generic",
+ "publish/" => "pubsub",
+ "pubsub-channels/" => "pubsub",
+ "pubsub-numpat/" => "pubsub",
+ "pubsub-numsub/" => "pubsub",
+ "pubsub-shardchannels/" => "pubsub",
+ "pubsub-shardnumsub/" => "pubsub",
+ "punsubscribe/" => "pubsub",
+ "quit/" => "connection",
+ "randomkey/" => "generic",
+ "readonly/" => "cluster",
+ "readwrite/" => "cluster",
+ "rename/" => "generic",
+ "renamenx/" => "generic",
+ "replconf/" => "server",
+ "replicaof/" => "server",
+ "reset/" => "connection",
+ "restore/" => "generic",
+ "restore-asking/" => "server",
+ "role/" => "server",
+ "rpop/" => "list",
+ "rpoplpush/" => "list",
+ "rpush/" => "list",
+ "rpushx/" => "list",
+ "sadd/" => "set",
+ "save/" => "server",
+ "scan/" => "generic",
+ "scard/" => "set",
+ "script-debug/" => "scripting",
+ "script-exists/" => "scripting",
+ "script-flush/" => "scripting",
+ "script-kill/" => "scripting",
+ "script-load/" => "scripting",
+ "sdiff/" => "set",
+ "sdiffstore/" => "set",
+ "select/" => "connection",
+ "set/" => "string",
+ "setbit/" => "bitmap",
+ "setex/" => "string",
+ "setnx/" => "string",
+ "setrange/" => "string",
+ "shutdown/" => "server",
+ "sinter/" => "set",
+ "sintercard/" => "set",
+ "sinterstore/" => "set",
+ "sismember/" => "set",
+ "slaveof/" => "server",
+ "slowlog-get/" => "server",
+ "slowlog-len/" => "server",
+ "slowlog-reset/" => "server",
+ "smembers/" => "set",
+ "smismember/" => "set",
+ "smove/" => "set",
+ "sort/" => "generic",
+ "sort_ro/" => "generic",
+ "spop/" => "set",
+ "spublish/" => "pubsub",
+ "srandmember/" => "set",
+ "srem/" => "set",
+ "sscan/" => "set",
+ "ssubscribe/" => "pubsub",
+ "strlen/" => "string",
+ "subscribe/" => "pubsub",
+ "substr/" => "string",
+ "sunion/" => "set",
+ "sunionstore/" => "set",
+ "sunsubscribe/" => "pubsub",
+ "swapdb/" => "server",
+ "sync/" => "server",
+ "tdigest.add/" => "tdigest",
+ "tdigest.byrank/" => "tdigest",
+ "tdigest.byrevrank/" => "tdigest",
+ "tdigest.cdf/" => "tdigest",
+ "tdigest.create/" => "tdigest",
+ "tdigest.info/" => "tdigest",
+ "tdigest.max/" => "tdigest",
+ "tdigest.merge/" => "tdigest",
+ "tdigest.min/" => "tdigest",
+ "tdigest.quantile/" => "tdigest",
+ "tdigest.rank/" => "tdigest",
+ "tdigest.reset/" => "tdigest",
+ "tdigest.revrank/" => "tdigest",
+ "tdigest.trimmed_mean/" => "tdigest",
+ "time/" => "server",
+ "topk.add/" => "topk",
+ "topk.count/" => "topk",
+ "topk.incrby/" => "topk",
+ "topk.info/" => "topk",
+ "topk.list/" => "topk",
+ "topk.query/" => "topk",
+ "topk.reserve/" => "topk",
+ "touch/" => "generic",
+ "ts.add/" => "timeseries",
+ "ts.alter/" => "timeseries",
+ "ts.create/" => "timeseries",
+ "ts.createrule/" => "timeseries",
+ "ts.decrby/" => "timeseries",
+ "ts.del/" => "timeseries",
+ "ts.deleterule/" => "timeseries",
+ "ts.get/" => "timeseries",
+ "ts.incrby/" => "timeseries",
+ "ts.info/" => "timeseries",
+ "ts.madd/" => "timeseries",
+ "ts.mget/" => "timeseries",
+ "ts.mrange/" => "timeseries",
+ "ts.mrevrange/" => "timeseries",
+ "ts.queryindex/" => "timeseries",
+ "ts.range/" => "timeseries",
+ "ts.revrange/" => "timeseries",
+ "ttl/" => "generic",
+ "type/" => "generic",
+ "unlink/" => "generic",
+ "unsubscribe/" => "pubsub",
+ "unwatch/" => "transactions",
+ "wait/" => "generic",
+ "watch/" => "transactions",
+ "xack/" => "stream",
+ "xadd/" => "stream",
+ "xautoclaim/" => "stream",
+ "xclaim/" => "stream",
+ "xdel/" => "stream",
+ "xgroup-create/" => "stream",
+ "xgroup-createconsumer/" => "stream",
+ "xgroup-delconsumer/" => "stream",
+ "xgroup-destroy/" => "stream",
+ "xgroup-setid/" => "stream",
+ "xinfo-consumers/" => "stream",
+ "xinfo-groups/" => "stream",
+ "xinfo-stream/" => "stream",
+ "xlen/" => "stream",
+ "xpending/" => "stream",
+ "xrange/" => "stream",
+ "xread/" => "stream",
+ "xreadgroup/" => "stream",
+ "xrevrange/" => "stream",
+ "xsetid/" => "stream",
+ "xtrim/" => "stream",
+ "zadd/" => "sorted-set",
+ "zcard/" => "sorted-set",
+ "zcount/" => "sorted-set",
+ "zdiff/" => "sorted-set",
+ "zdiffstore/" => "sorted-set",
+ "zincrby/" => "sorted-set",
+ "zinter/" => "sorted-set",
+ "zintercard/" => "sorted-set",
+ "zinterstore/" => "sorted-set",
+ "zlexcount/" => "sorted-set",
+ "zmpop/" => "sorted-set",
+ "zmscore/" => "sorted-set",
+ "zpopmax/" => "sorted-set",
+ "zpopmin/" => "sorted-set",
+ "zrandmember/" => "sorted-set",
+ "zrange/" => "sorted-set",
+ "zrangebylex/" => "sorted-set",
+ "zrangebyscore/" => "sorted-set",
+ "zrangestore/" => "sorted-set",
+ "zrank/" => "sorted-set",
+ "zrem/" => "sorted-set",
+ "zremrangebylex/" => "sorted-set",
+ "zremrangebyrank/" => "sorted-set",
+ "zremrangebyscore/" => "sorted-set",
+ "zrevrange/" => "sorted-set",
+ "zrevrangebylex/" => "sorted-set",
+ "zrevrangebyscore/" => "sorted-set",
+ "zrevrank/" => "sorted-set",
+ "zscan/" => "sorted-set",
+ "zscore/" => "sorted-set",
+ "zunion/" => "sorted-set",
+ "zunionstore/" => "sorted-set"
+ }
+
+ SLUG_NAMES = {
+ "bitmap" => "Core: Bitmap",
+ "cluster" => "Core: Cluster management",
+ "connection" => "Core: Connection management",
+ "generic" => "Core: Generic",
+ "geo" => "Core: Geospatial indices",
+ "hash" => "Core: Hash",
+ "hyperloglog" => "Core: HyperLogLog",
+ "list" => "Core: List",
+ "pubsub" => "Core: Pub/Sub",
+ "scripting" => "Core: Scripting and Functions",
+ "server" => "Core: Server managment",
+ "set" => "Core: Set",
+ "sorted-set:" => "Core: Sorted Set",
+ "stream" => "Core: Stream",
+ "string" => "Core: String",
+ "transactions" => "Core: Transactions",
+ "bf" => "Stack: Bloom Filter",
+ "cf" => "Stack: Cuckoo Filter",
+ "cms" => "Stack: Count-min Sketch",
+ "graph" => "Stack: Graph",
+ "json" => "Stack: JSON",
+ "search" => "Stack: Search",
+ "suggestion" => "Stack: Auto-Suggest",
+ "tdigest" => "Stack: T-Digest",
+ "timeseries" => "Stack: Time Series",
+ "topk" => "Stack: Top-K",
+ }
+
end
end
end
diff --git a/lib/docs/scrapers/redis.rb b/lib/docs/scrapers/redis.rb
index cffe2b1f..46b299af 100644
--- a/lib/docs/scrapers/redis.rb
+++ b/lib/docs/scrapers/redis.rb
@@ -1,7 +1,7 @@
module Docs
class Redis < UrlScraper
self.type = 'redis'
- self.release = '6.2.1'
+ self.release = '7.0.8'
self.base_url = 'https://redis.io/commands'
self.links = {
home: 'https://redis.io/',
@@ -10,13 +10,13 @@ module Docs
html_filters.push 'redis/entries', 'redis/clean_html', 'title'
- options[:container] = ->(filter) { filter.root_page? ? '#commands' : '.text' }
+ options[:container] = ->(filter) { filter.root_page? ? '#commands-grid' : 'section' }
options[:title] = false
options[:root_title] = 'Redis'
options[:follow_links] = ->(filter) { filter.root_page? }
options[:attribution] = <<-HTML
- © 2009–2020 Salvatore Sanfilippo
+ © 2009–2022 Salvatore Sanfilippo
Licensed under the Creative Commons Attribution-ShareAlike License 4.0.
HTML
@@ -25,5 +25,13 @@ module Docs
body = body.lines[1..-1].join
body.scan(/Redis ([0-9.]+)/)[0][0]
end
+
+ private
+
+ def parse(response)
+ response.body.gsub! '