From e27ead7c64715f7bfa4b1c613ba08bc71ede21ca Mon Sep 17 00:00:00 2001
From: arkcia <arkcia@126.com>
Date: Sun, 21 Jan 2024 11:06:16 +0800
Subject: [PATCH 1/2] update and fix pytroch
---
lib/docs/filters/pytorch/entries.rb | 120 +++++++++++++++++++++-------
lib/docs/scrapers/pytorch.rb | 12 ++-
2 files changed, 94 insertions(+), 38 deletions(-)
diff --git a/lib/docs/filters/pytorch/entries.rb b/lib/docs/filters/pytorch/entries.rb
index 4b8d153e..b337edbb 100644
--- a/lib/docs/filters/pytorch/entries.rb
+++ b/lib/docs/filters/pytorch/entries.rb
@@ -1,57 +1,115 @@
module Docs
class Pytorch
class EntriesFilter < Docs::EntriesFilter
- NAME_REPLACEMENTS = {
- "Distributed communication package - torch.distributed" => "torch.distributed"
+ TYPE_REPLACEMENTS = {
+ "torch.Tensor" => "Tensor",
+ "torch.nn" => "Neuro Network",
+ "Probability distributions - torch.distributions" => "Probability Distributions",
+ "torch" => "Torch",
+ "Quantization" => "Quantization",
+ "torch.optim" => "Optimization",
+ "torch.Storage" => "Storage",
+ "torch.nn.functional" => "NN Functions",
+ "torch.cuda" => "CUDA",
+ "Torch Distributed Elastic" => "Distributed Elastic",
+ "torch.fx" => "FX",
+ "TorchScript" => "Torch Script",
+ "torch.onnx" => "ONNX",
+ "Distributed communication package - torch.distributed" => "Distributed Communication",
+ "Automatic differentiation package - torch.autograd" => "Automatic Differentiation",
+ "torch.linalg" => "Linear Algebra",
+ "Distributed Checkpoint - torch.distributed.checkpoint" => "Distributed Checkpoint",
+ "Distributed RPC Framework" => "Distributed RPC",
+ "torch.special" => "SciPy-like Special",
+ "torch.package" => "Package",
+ "torch.backends" => "Backends",
+ "FullyShardedDataParallel" => "Fully Sharded Data Parallel",
+ "torch.sparse" => "Sparse Tensors",
+ "torch.export" => "Traced Graph Export",
+ "torch.fft" => "Discrete Fourier Transforms",
+ "torch.utils.data" => "Datasets and Data Loaders",
+ "torch.monitor" => "Monitor",
+ "Automatic Mixed Precision package - torch.amp" => "Automatic Mixed Precision",
+ "torch.utils.tensorboard" => "Tensorboard",
+ "torch.profiler" => "Profiler",
+ "torch.mps" => "MPS",
+ "DDP Communication Hooks" => "DDP Communication Hooks",
+ "Benchmark Utils - torch.utils.benchmark" => "Benchmark Utils",
+ "torch.nn.init" => "Parameter Initializations",
+ "Tensor Parallelism - torch.distributed.tensor.parallel" => "Tensor Parallelism",
+ "torch.func" => "JAX-like Function Transforms",
+ "Distributed Optimizers" => "Distributed Optimizers",
+ "torch.signal" => "SciPy-like Signal",
+ "torch.futures" => "Miscellaneous",
+ "torch.utils.cpp_extension" => "Miscellaneous",
+ "torch.overrides" => "Miscellaneous",
+ "Generic Join Context Manager" => "Miscellaneous",
+ "torch.hub" => "Miscellaneous",
+ "torch.cpu" => "Miscellaneous",
+ "torch.random" => "Miscellaneous",
+ "torch.compiler" => "Miscellaneous",
+ "Pipeline Parallelism" => "Miscellaneous",
+ "Named Tensors" => "Miscellaneous",
+ "Multiprocessing package - torch.multiprocessing" => "Miscellaneous",
+ "torch.utils" => "Miscellaneous",
+ "torch.library" => "Miscellaneous",
+ "Tensor Attributes" => "Miscellaneous",
+ "torch.testing" => "Miscellaneous",
+ "torch.nested" => "Miscellaneous",
+ "Understanding CUDA Memory Usage" => "Miscellaneous",
+ "torch.utils.dlpack" => "Miscellaneous",
+ "torch.utils.checkpoint" => "Miscellaneous",
+ "torch.__config__" => "Miscellaneous",
+ "Type Info" => "Miscellaneous",
+ "torch.utils.model_zoo" => "Miscellaneous",
+ "torch.utils.mobile_optimizer" => "Miscellaneous",
+ "torch._logging" => "Miscellaneous",
+ "torch.masked" => "Miscellaneous",
+ "torch.utils.bottleneck" => "Miscellaneous"
}
- def get_breadcrumbs()
- css('.pytorch-breadcrumbs > li').map { |node| node.content.delete_suffix(' >') }
+ def get_breadcrumbs
+ css('.pytorch-breadcrumbs > li').map {
+ |node| node.content.delete_suffix(' >').strip
+ }.reject { |item| item.nil? || item.empty? }
end
def get_name
- # The id of the container `div.section` indicates the page type.
- # If the id starts with `module-`, then it's an API reference,
- # otherwise it is a note or design doc.
- section_id = at_css('.section[id], section[id]')['id']
- if section_id.starts_with? 'module-'
- section_id.remove('module-')
- else
- name = get_breadcrumbs()[1]
- NAME_REPLACEMENTS.fetch(name, name)
- end
+ b = get_breadcrumbs
+ b[(b[1] == 'torch' ? 2 : 1)..].join('.')
end
def get_type
- name
+ t = get_breadcrumbs[1]
+ TYPE_REPLACEMENTS.fetch(t, t)
end
def include_default_entry?
- # Only include API references, and ignore notes or design docs
- !subpath.start_with? 'generated/' and type.start_with? 'torch'
+ # Only include API entries to simplify and unify the list
+ return name.start_with?('torch.')
end
def additional_entries
return [] if root_page?
entries = []
-
- css('dt').each do |node|
- name = node['id']
- if name == self.name or name == nil
+ css('dl').each do |node|
+ dt = node.at_css('dt')
+ if dt == nil
+ next
+ end
+ id = dt['id']
+ if id == name or id == nil
next
end
- case node.parent['class']
- when 'method', 'function'
- if node.at_css('code').content.starts_with? 'property '
- # this instance method is a property, so treat it as an attribute
- entries << [name, node['id']]
- else
- entries << [name + '()', node['id']]
- end
- when 'class', 'attribute'
- entries << [name, node['id']]
+ case node['class']
+ when 'py method', 'py function'
+ entries << [id + '()', id]
+ when 'py class', 'py attribute', 'py property'
+ entries << [id, id]
+ when 'footnote brackets', 'field-list simple'
+ next
end
end
diff --git a/lib/docs/scrapers/pytorch.rb b/lib/docs/scrapers/pytorch.rb
index a6d9ea82..8f560a94 100644
--- a/lib/docs/scrapers/pytorch.rb
+++ b/lib/docs/scrapers/pytorch.rb
@@ -3,7 +3,6 @@ module Docs
self.name = 'PyTorch'
self.slug = 'pytorch'
self.type = 'sphinx'
- self.force_gzip = true
self.links = {
home: 'https://pytorch.org/',
code: 'https://github.com/pytorch/pytorch'
@@ -11,21 +10,20 @@ module Docs
html_filters.push 'pytorch/entries', 'pytorch/clean_html', 'sphinx/clean_html'
- options[:skip] = ['cpp_index.html', 'packages.html', 'py-modindex.html', 'genindex.html']
+ options[:skip] = ['cpp_index.html', 'deploy.html', 'packages.html', 'py-modindex.html', 'genindex.html']
options[:skip_patterns] = [/\Acommunity/, /\A_modules/, /\Anotes/, /\Aorg\/pytorch\//]
- options[:max_image_size] = 256_000
options[:attribution] = <<-HTML
- © 2019-2024 Torch Contributors<br>
- Licensed under the 3-clause BSD License.
+ © 2024, PyTorch Contributors<br>
+ PyTorch has a BSD-style license, as found in the <a href="https://github.com/pytorch/pytorch/blob/main/LICENSE">LICENSE</a> file.
HTML
- version '2' do
+ version '2.1' do
self.release = '2.1'
self.base_url = "https://pytorch.org/docs/#{release}/"
end
- version '1' do
+ version '1.13' do
self.release = '1.13'
self.base_url = "https://pytorch.org/docs/#{release}/"
end
From d9e4d516b8a7142bafccaa429df13dd9bf9ca5d2 Mon Sep 17 00:00:00 2001
From: Simon Legner <Simon.Legner@gmail.com>
Date: Sun, 21 Jan 2024 20:36:23 +0100
Subject: [PATCH 2/2] pytorch: version 2 and version 1
---
lib/docs/scrapers/pytorch.rb | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/docs/scrapers/pytorch.rb b/lib/docs/scrapers/pytorch.rb
index 8f560a94..ae79ec36 100644
--- a/lib/docs/scrapers/pytorch.rb
+++ b/lib/docs/scrapers/pytorch.rb
@@ -18,12 +18,12 @@ module Docs
PyTorch has a BSD-style license, as found in the <a href="https://github.com/pytorch/pytorch/blob/main/LICENSE">LICENSE</a> file.
HTML
- version '2.1' do
+ version '2' do
self.release = '2.1'
self.base_url = "https://pytorch.org/docs/#{release}/"
end
- version '1.13' do
+ version '1' do
self.release = '1.13'
self.base_url = "https://pytorch.org/docs/#{release}/"
end