diff --git a/lib/docs/filters/pytorch/entries.rb b/lib/docs/filters/pytorch/entries.rb index 4b8d153e..b337edbb 100644 --- a/lib/docs/filters/pytorch/entries.rb +++ b/lib/docs/filters/pytorch/entries.rb @@ -1,57 +1,115 @@ module Docs class Pytorch class EntriesFilter < Docs::EntriesFilter - NAME_REPLACEMENTS = { - "Distributed communication package - torch.distributed" => "torch.distributed" + TYPE_REPLACEMENTS = { + "torch.Tensor" => "Tensor", + "torch.nn" => "Neuro Network", + "Probability distributions - torch.distributions" => "Probability Distributions", + "torch" => "Torch", + "Quantization" => "Quantization", + "torch.optim" => "Optimization", + "torch.Storage" => "Storage", + "torch.nn.functional" => "NN Functions", + "torch.cuda" => "CUDA", + "Torch Distributed Elastic" => "Distributed Elastic", + "torch.fx" => "FX", + "TorchScript" => "Torch Script", + "torch.onnx" => "ONNX", + "Distributed communication package - torch.distributed" => "Distributed Communication", + "Automatic differentiation package - torch.autograd" => "Automatic Differentiation", + "torch.linalg" => "Linear Algebra", + "Distributed Checkpoint - torch.distributed.checkpoint" => "Distributed Checkpoint", + "Distributed RPC Framework" => "Distributed RPC", + "torch.special" => "SciPy-like Special", + "torch.package" => "Package", + "torch.backends" => "Backends", + "FullyShardedDataParallel" => "Fully Sharded Data Parallel", + "torch.sparse" => "Sparse Tensors", + "torch.export" => "Traced Graph Export", + "torch.fft" => "Discrete Fourier Transforms", + "torch.utils.data" => "Datasets and Data Loaders", + "torch.monitor" => "Monitor", + "Automatic Mixed Precision package - torch.amp" => "Automatic Mixed Precision", + "torch.utils.tensorboard" => "Tensorboard", + "torch.profiler" => "Profiler", + "torch.mps" => "MPS", + "DDP Communication Hooks" => "DDP Communication Hooks", + "Benchmark Utils - torch.utils.benchmark" => "Benchmark Utils", + "torch.nn.init" => "Parameter Initializations", + "Tensor Parallelism - torch.distributed.tensor.parallel" => "Tensor Parallelism", + "torch.func" => "JAX-like Function Transforms", + "Distributed Optimizers" => "Distributed Optimizers", + "torch.signal" => "SciPy-like Signal", + "torch.futures" => "Miscellaneous", + "torch.utils.cpp_extension" => "Miscellaneous", + "torch.overrides" => "Miscellaneous", + "Generic Join Context Manager" => "Miscellaneous", + "torch.hub" => "Miscellaneous", + "torch.cpu" => "Miscellaneous", + "torch.random" => "Miscellaneous", + "torch.compiler" => "Miscellaneous", + "Pipeline Parallelism" => "Miscellaneous", + "Named Tensors" => "Miscellaneous", + "Multiprocessing package - torch.multiprocessing" => "Miscellaneous", + "torch.utils" => "Miscellaneous", + "torch.library" => "Miscellaneous", + "Tensor Attributes" => "Miscellaneous", + "torch.testing" => "Miscellaneous", + "torch.nested" => "Miscellaneous", + "Understanding CUDA Memory Usage" => "Miscellaneous", + "torch.utils.dlpack" => "Miscellaneous", + "torch.utils.checkpoint" => "Miscellaneous", + "torch.__config__" => "Miscellaneous", + "Type Info" => "Miscellaneous", + "torch.utils.model_zoo" => "Miscellaneous", + "torch.utils.mobile_optimizer" => "Miscellaneous", + "torch._logging" => "Miscellaneous", + "torch.masked" => "Miscellaneous", + "torch.utils.bottleneck" => "Miscellaneous" } - def get_breadcrumbs() - css('.pytorch-breadcrumbs > li').map { |node| node.content.delete_suffix(' >') } + def get_breadcrumbs + css('.pytorch-breadcrumbs > li').map { + |node| node.content.delete_suffix(' >').strip + }.reject { |item| item.nil? || item.empty? } end def get_name - # The id of the container `div.section` indicates the page type. - # If the id starts with `module-`, then it's an API reference, - # otherwise it is a note or design doc. - section_id = at_css('.section[id], section[id]')['id'] - if section_id.starts_with? 'module-' - section_id.remove('module-') - else - name = get_breadcrumbs()[1] - NAME_REPLACEMENTS.fetch(name, name) - end + b = get_breadcrumbs + b[(b[1] == 'torch' ? 2 : 1)..].join('.') end def get_type - name + t = get_breadcrumbs[1] + TYPE_REPLACEMENTS.fetch(t, t) end def include_default_entry? - # Only include API references, and ignore notes or design docs - !subpath.start_with? 'generated/' and type.start_with? 'torch' + # Only include API entries to simplify and unify the list + return name.start_with?('torch.') end def additional_entries return [] if root_page? entries = [] - - css('dt').each do |node| - name = node['id'] - if name == self.name or name == nil + css('dl').each do |node| + dt = node.at_css('dt') + if dt == nil + next + end + id = dt['id'] + if id == name or id == nil next end - case node.parent['class'] - when 'method', 'function' - if node.at_css('code').content.starts_with? 'property ' - # this instance method is a property, so treat it as an attribute - entries << [name, node['id']] - else - entries << [name + '()', node['id']] - end - when 'class', 'attribute' - entries << [name, node['id']] + case node['class'] + when 'py method', 'py function' + entries << [id + '()', id] + when 'py class', 'py attribute', 'py property' + entries << [id, id] + when 'footnote brackets', 'field-list simple' + next end end diff --git a/lib/docs/scrapers/pytorch.rb b/lib/docs/scrapers/pytorch.rb index a6d9ea82..8f560a94 100644 --- a/lib/docs/scrapers/pytorch.rb +++ b/lib/docs/scrapers/pytorch.rb @@ -3,7 +3,6 @@ module Docs self.name = 'PyTorch' self.slug = 'pytorch' self.type = 'sphinx' - self.force_gzip = true self.links = { home: 'https://pytorch.org/', code: 'https://github.com/pytorch/pytorch' @@ -11,21 +10,20 @@ module Docs html_filters.push 'pytorch/entries', 'pytorch/clean_html', 'sphinx/clean_html' - options[:skip] = ['cpp_index.html', 'packages.html', 'py-modindex.html', 'genindex.html'] + options[:skip] = ['cpp_index.html', 'deploy.html', 'packages.html', 'py-modindex.html', 'genindex.html'] options[:skip_patterns] = [/\Acommunity/, /\A_modules/, /\Anotes/, /\Aorg\/pytorch\//] - options[:max_image_size] = 256_000 options[:attribution] = <<-HTML - © 2019-2024 Torch Contributors
- Licensed under the 3-clause BSD License. + © 2024, PyTorch Contributors
+ PyTorch has a BSD-style license, as found in the LICENSE file. HTML - version '2' do + version '2.1' do self.release = '2.1' self.base_url = "https://pytorch.org/docs/#{release}/" end - version '1' do + version '1.13' do self.release = '1.13' self.base_url = "https://pytorch.org/docs/#{release}/" end