From f1dd2acc0851267b280504d971d8bd37e97681f1 Mon Sep 17 00:00:00 2001 From: Thomas Dy Date: Sun, 24 Apr 2022 18:28:13 +0900 Subject: [PATCH] Add nix and nixpkgs This only adds a subset of nix and nixpkgs functions. Notably derivation, stdenv and language-specific functions are omitted for now as their formats differ significantly and will require more cleanup. --- .../templates/pages/about_tmpl.coffee | 5 + lib/docs/filters/nix/clean_html.rb | 113 ++++++++++++++++++ lib/docs/filters/nix/entries.rb | 22 ++++ lib/docs/scrapers/nix.rb | 31 +++++ public/icons/docs/nix/16.png | Bin 0 -> 1220 bytes public/icons/docs/nix/16@2x.png | Bin 0 -> 1936 bytes public/icons/docs/nix/SOURCE | 1 + 7 files changed, 172 insertions(+) create mode 100644 lib/docs/filters/nix/clean_html.rb create mode 100644 lib/docs/filters/nix/entries.rb create mode 100644 lib/docs/scrapers/nix.rb create mode 100644 public/icons/docs/nix/16.png create mode 100644 public/icons/docs/nix/16@2x.png create mode 100644 public/icons/docs/nix/SOURCE diff --git a/assets/javascripts/templates/pages/about_tmpl.coffee b/assets/javascripts/templates/pages/about_tmpl.coffee index a366449a..6fdba573 100644 --- a/assets/javascripts/templates/pages/about_tmpl.coffee +++ b/assets/javascripts/templates/pages/about_tmpl.coffee @@ -587,6 +587,11 @@ credits = [ '2006-2021 Andreas Rumpf', 'MIT', 'https://github.com/nim-lang/Nim#license' + ], [ + 'Nix', + '2022 NixOS Contributors', + 'LGPLv2.1', + 'https://github.com/NixOS/nix#license' ], [ 'Node.js', 'Joyent, Inc. and other Node contributors
Node.js is a trademark of Joyent, Inc.', diff --git a/lib/docs/filters/nix/clean_html.rb b/lib/docs/filters/nix/clean_html.rb new file mode 100644 index 00000000..731fc0cf --- /dev/null +++ b/lib/docs/filters/nix/clean_html.rb @@ -0,0 +1,113 @@ +module Docs + class Nix + class CleanHtmlFilter < Filter + def call + if subpath == 'nixpkgs/stable/index.html' + new_root = Nokogiri::XML::Node.new 'div', doc.document + + # lib functions + lib_sections = xpath("//*[@id='sec-functions-library']/ancestor::section[1]/section/section") + lib_sections.css('.titlepage .title').each do |title| + title.name = 'h2' + strip_section_number(title.children) + title['id'] = title.content.gsub(/[^a-zA-Z0-9]/, '-') + end + lib_sections.css('.example .title strong').each do |title| + title.content = title.content.sub(/^Example ([0-9.]+)/, 'Example: ') + end + new_root.add_child(lib_sections) + + # fetchers + fetcher_sections = xpath("//*[@id='chap-pkgs-fetchers']/ancestor::section[1]/section[position()>1]") + fetcher_sections.css('.titlepage .title').each do |title| + strip_section_number(title.children) + prefix_with(title, 'pkgs') if title.name == 'h2' + end + new_root.add_child(fetcher_sections) + + # trivial builders + trivial_sections = xpath("//*[@id='chap-trivial-builders']/ancestor::section[1]/section") + trivial_sections.css('.titlepage .title').each do |title| + strip_section_number(title.children) + prefix_with(title, 'pkgs') if title.name == 'h2' + end + new_root.add_child(trivial_sections) + + # special builders + special_sections = xpath("//*[@id='chap-special']/ancestor::section[1]/section") + special_sections.css('.titlepage .title').each do |title| + strip_section_number(title.children) + if title.name == 'h2' + title.children[0].wrap('') + prefix_with(title, 'pkgs') + end + end + new_root.add_child(special_sections) + + # image builders + image_sections = xpath("//*[@id='chap-images']/ancestor::section[1]/section") + image_sections.css('.titlepage .title').each do |title| + strip_section_number(title.children) + title.children[0].wrap('') + end + image_sections.each do |section| + prefix = section.at_xpath('*[@class="titlepage"]//*[@class="title"]').content + next unless ["pkgs.dockerTools", "pkgs.ociTools"].include?(prefix) + section.xpath('section/*[@class="titlepage"]//*[@class="title"]').each do |title| + prefix_with(title, prefix) + title['data-add-to-index'] = '' + end + end + new_root.add_child(image_sections) + + new_root + elsif subpath == 'nix/stable/expressions/builtins.html' + @doc = doc.at_css('main dl') + + # strip out the first entry, `derivation`, the actual documentation + # exists in a separate page + derivation_dt = doc.children.at_css('dt') + if derivation_dt.content.starts_with?('derivation') + derivation_dt.remove + doc.children.at_css('dd').remove + else + raise RuntimeError.new('First entry is not derivation, update the scraper') + end + + doc.css('dt').each do |title| + title.name = 'h2' + unwrap(title.at_css('a')) + title.children[0].children.before('builtins.') + end + doc.css('dd').each do |description| + description.name = 'div' + end + + doc + else + doc + end + end + + def strip_section_number(title_children) + while title_children.first.content == '' + title_children.shift.remove + end + first_text = title_children.first + return unless first_text.text? + first_text.content = first_text.content.sub(/[0-9.]+ ?/, '') + first_text.remove if first_text.blank? + end + + def prefix_with(title_node, text) + title_node.css('code').each do |code| + code.content = "#{text}.#{code.content}" unless code.content.starts_with?("#{text}.") + end + end + + def unwrap(node) + node.replace(node.inner_html) + end + end + end +end diff --git a/lib/docs/filters/nix/entries.rb b/lib/docs/filters/nix/entries.rb new file mode 100644 index 00000000..f4ff093a --- /dev/null +++ b/lib/docs/filters/nix/entries.rb @@ -0,0 +1,22 @@ +module Docs + class Nix + class EntriesFilter < Docs::EntriesFilter + def include_default_entry? + false + end + + def additional_entries + css('h2, h3[data-add-to-index]').flat_map do |node| + node.css('code').map do |code| + title = code.content + index = title.rindex('.') + type = title[0...index] + name = title.match(/^[^\s]+/)[0] + + [name, node['id'], type] + end + end + end + end + end +end diff --git a/lib/docs/scrapers/nix.rb b/lib/docs/scrapers/nix.rb new file mode 100644 index 00000000..ae549818 --- /dev/null +++ b/lib/docs/scrapers/nix.rb @@ -0,0 +1,31 @@ +module Docs + class Nix < UrlScraper + self.type = 'simple' + self.release = '2.8.0' + self.base_url = 'https://nixos.org/manual/' + self.root_path = 'nix/stable/expressions/builtins.html' + self.initial_paths = %w( + nix/stable/expressions/builtins.html + nixpkgs/stable/index.html) + self.links = { + home: 'https://nixos.org/', + code: 'https://github.com/NixOS/nix' + } + + html_filters.push 'nix/clean_html', 'nix/entries' + + options[:skip_links] = true + + options[:attribution] = <<-HTML + © 2022 NixOS Contributors
+ Licensed under the LGPL License. + HTML + + def get_latest_version(opts) + doc = fetch_doc('https://nixos.org/manual/nix/stable/', opts) + json = JSON.parse(doc.at_css('body')['data-nix-channels']) + channel = json.find { |c| c['channel'] == 'stable' } + channel['version'] + end + end +end diff --git a/public/icons/docs/nix/16.png b/public/icons/docs/nix/16.png new file mode 100644 index 0000000000000000000000000000000000000000..13ad550b0a98bde25c53fe022ff7a4aa712d971a GIT binary patch literal 1220 zcmeAS@N?(olHy`uVBq!ia0vp^0zk~i!3-n?Kj!QJQY`6?zK#qG8~eHcB(ehe3dtTp zz6=aiY77hwEes65fID z(nEn|heN84m9ZrLHPkd>QDG`YrcEDvv-^)^2;)dhq%G|Nk3zJOql@ zZN9hs<^QQyzt?TLSHALQNF~tHLqQcsppM-B2xw8=)(7=l9{|H7w(WRe=>ec)fJVVR zl0D~~d-^J$ysf?k+x?1m1r+ZMDBTZK0}qPotq%iBw|V7kbc|okz!2e;wE^fhuk6hX z44Iyp+XG7XK~&c4c;ub4$tQnnSk3;rP4^qNKFFPWE}~&iO#7i~pl5eKt=|IUKN=`FGcoAduG%!Gj&-O<;6@rGVxG4FswTDA@}Py#;f=gA&>9k|4ieAdMaPzxOWr z6Yu&iK>fj14)OCB4n5Iz$vOJFFMa8sDsD!_3*SF|`S$JposZ@`leLAc?3SMmw=|FZ zG%@u@6bn0}&D({{%yaMEKgT@p!0y{m^|g<@IJ9`l(HBQ`bRXZ{duS1h!r2FR7ceu; zduPwY&KmI}eZr?03#;(+tL?3Yw5Rc!ez^DG+xM@Zzbi5Fl>J?v*83~RMf>5g3lbb# z9%}f$_lo;d?)5%kkH5uRpxYUfyxmlFz z;kcfhkdQEC%A~YO6Q@oN4@gO4D3X+%IwdsZ>J?Lyty>}`BpaEmc2)giYjZoc+c(1Z zYy;CXv1wt~%(j(%*}?u z<|byQ#vI$XY};C1Qdau?%eSxoGc*n<*kpd1vhwn>^D`{7zs=bx^|?i|<+;8F_2{gvudn6b(BKhCND#QcrSkN(+}o;UZ|<-@VPX(j<8kn{ zURWA103=IXBT9nv(@M${i&7cN%ggmL^RkPR6AM!H@{7`Ezq647Dq`?-^>bP0l+XkK DMx004R>004l5008;`004mK004C`008P>0026e000+ooVrmw00006 zVoOIv0RI600RN!9r;`8x010qNS#tmYAff;OAff@)a0gZZ00z%VL_t(Y$9uVwag4pjO9ZdlkxLfL23P<*2{Zu_xEt7}rT0eHRiNe`ut9)O#GZ;=LQUxkI!gK#mhaKsg3 zo3ia;+ry+OH)0Dz{R3Y<^wEL+!Y9(;a|6*A5KZZm9)MQhUf>qs#@ffLE>Otfm>d9B zRX}V!u0G$=G@*D!w2LvZ2Koj+JbCQI9)xfuh!j3MQ2KPn#WMzh71%}0eZY&8&Tc&1 zefZ>efh)71Uj;k^^o}sVF#&iz8%Qv|wT|;Ho_tS?Uh3#vx$Mu3HXhp3`OjmW$5VQ; z=iAPo2BsH04>JWkS8#jx;gj2dHJr-0-4z!AX#H~ss$2ue#?fSXJR7_dGb?4pkH)g^ zL&W~R$j@#|1AqOq`yq)?EQrCujhG*2`u}Nvv-92ZP`o1p=>Wb7d|-?a0aZd#nHhp0 zoXZKZ25A-&h<%cNnJa{7Qw0auaQ%#QGB9Y&X6!wVrl+f~Yj|LIi5*ji!e%>uKn7lM z!KIj+!TBnqCJHyF>6X!gHY3^r@`W=CsB;0ug#MFpZ=tBo<)Qff!yk7y+i?mAtU;tp zg&s*AOc~gw_0J!C9{akQ{LS9z>RHn33R;TY70U@5mYmOMvi_-k4D?5eb%lu?yFY!P zx9h|jqy{OaSqdPfyO#i7%>(t>Vnps$i=-GRb(CIv|8H+64HKFu)-^Gfy!GO4K-ehc z!sSwE$aYT^qdFHtNgh}SdKlBkzV+e5TgD>;M zL-R*~FTQstK{=umByPjL+C*gaT49@@*Tf)6c90-3{$n0^EgLEA(pq47=8K~TK0Mg% z7_JDp++B5z<95V^Jtunx&e(l2)Q@uokY)+3cFH%PCnT)~MJf?5s*HqF2?$8Dfz?1j zVtA;$^LS_Pl#t5@Da-<`8P`}J5G=p{r7HOKKm0RNAST0djH~Q8+l~-phaDe{NjWCv zArt~qNMj(2F#)W9{vf1{$0pLw24J708{XXU=29sDdh3w7iEqL6QsLV{U;(KWMw+BB&d1)ag#h6U%0_!>3}5lGNe|dwJmKEnt-z_Wdp%kUz!>Vb;S$&Kkt86A-tnpZg|4fmZZM9ex>aSdH~uH zuU2TUHQ;mLG2klTEnv&kS!Z$R!^52lVH4(Yq;a5yC%XHJjm^dRKt0uLH7!#o#);D= z#R@BiWi{XcgTTt#_U!bEvV*h>dITtS{araR z=_16`Y~6?SHfe9E?EDb$O2z-DyPzVS0HuB1>Gvq{JP~fL(3%;M?>#y&=YQ{%mR+`R zX8ZB3zJQ5{+kszH-)nCo=qXGdh=_sHyJL-F0Lvfv2Qf(injO6jv`DD|?UI^6(=*jg z-ADS?er4Xd9zUfKU_8>3h-)*0Ujmi`J*P+iSTRtQm>A3oN#BqHQs|R(Khh&LYTs*Z zYq)Cv?xS?=?**wB_&IPQyZ?34E~pAz3qH01hC(r)M_~~P1qFe$BbU$Z6xss(JJW2n zS*xIitnUMOf%K0F(%P~JC^CkOZh%CLDobI96w09Ek~Swv9Cq{(#BYHl(_Vx<&(W0< zI}G|Y(4FZ#4UG9?@_#8SBd~Y+peSaefwW^{L9a%BK_cXuvnZfkR6VQ^(G WZ*pgw?mQX*0000