update and fix pytroch

lib/docs/filters/pytorch/entries.rb

@@ -1,57 +1,115 @@
 module Docs
   class Pytorch
     class EntriesFilter < Docs::EntriesFilter
-      NAME_REPLACEMENTS = {
-        "Distributed communication package - torch.distributed" => "torch.distributed"
+      TYPE_REPLACEMENTS = {
+        "torch.Tensor" => "Tensor",
+        "torch.nn" => "Neuro Network",
+        "Probability distributions - torch.distributions" => "Probability Distributions",
+        "torch" => "Torch",
+        "Quantization" => "Quantization",
+        "torch.optim" => "Optimization",
+        "torch.Storage" => "Storage",
+        "torch.nn.functional" => "NN Functions",
+        "torch.cuda" => "CUDA",
+        "Torch Distributed Elastic" => "Distributed Elastic",
+        "torch.fx" => "FX",
+        "TorchScript" => "Torch Script",
+        "torch.onnx" => "ONNX",
+        "Distributed communication package - torch.distributed" => "Distributed Communication",
+        "Automatic differentiation package - torch.autograd" => "Automatic Differentiation",
+        "torch.linalg" => "Linear Algebra",
+        "Distributed Checkpoint - torch.distributed.checkpoint" => "Distributed Checkpoint",
+        "Distributed RPC Framework" => "Distributed RPC",
+        "torch.special" => "SciPy-like Special",
+        "torch.package" => "Package",
+        "torch.backends" => "Backends",
+        "FullyShardedDataParallel" => "Fully Sharded Data Parallel",
+        "torch.sparse" => "Sparse Tensors",
+        "torch.export" => "Traced Graph Export",
+        "torch.fft" => "Discrete Fourier Transforms",
+        "torch.utils.data" => "Datasets and Data Loaders",
+        "torch.monitor" => "Monitor",
+        "Automatic Mixed Precision package - torch.amp" => "Automatic Mixed Precision",
+        "torch.utils.tensorboard" => "Tensorboard",
+        "torch.profiler" => "Profiler",
+        "torch.mps" => "MPS",
+        "DDP Communication Hooks" => "DDP Communication Hooks",
+        "Benchmark Utils - torch.utils.benchmark" => "Benchmark Utils",
+        "torch.nn.init" => "Parameter Initializations",
+        "Tensor Parallelism - torch.distributed.tensor.parallel" => "Tensor Parallelism",
+        "torch.func" => "JAX-like Function Transforms",
+        "Distributed Optimizers" => "Distributed Optimizers",
+        "torch.signal" => "SciPy-like Signal",
+        "torch.futures" => "Miscellaneous",
+        "torch.utils.cpp_extension" => "Miscellaneous",
+        "torch.overrides" => "Miscellaneous",
+        "Generic Join Context Manager" => "Miscellaneous",
+        "torch.hub" => "Miscellaneous",
+        "torch.cpu" => "Miscellaneous",
+        "torch.random" => "Miscellaneous",
+        "torch.compiler" => "Miscellaneous",
+        "Pipeline Parallelism" => "Miscellaneous",
+        "Named Tensors" => "Miscellaneous",
+        "Multiprocessing package - torch.multiprocessing" => "Miscellaneous",
+        "torch.utils" => "Miscellaneous",
+        "torch.library" => "Miscellaneous",
+        "Tensor Attributes" => "Miscellaneous",
+        "torch.testing" => "Miscellaneous",
+        "torch.nested" => "Miscellaneous",
+        "Understanding CUDA Memory Usage" => "Miscellaneous",
+        "torch.utils.dlpack" => "Miscellaneous",
+        "torch.utils.checkpoint" => "Miscellaneous",
+        "torch.__config__" => "Miscellaneous",
+        "Type Info" => "Miscellaneous",
+        "torch.utils.model_zoo" => "Miscellaneous",
+        "torch.utils.mobile_optimizer" => "Miscellaneous",
+        "torch._logging" => "Miscellaneous",
+        "torch.masked" => "Miscellaneous",
+        "torch.utils.bottleneck" => "Miscellaneous"
       }
 
-      def get_breadcrumbs()
-        css('.pytorch-breadcrumbs > li').map { |node| node.content.delete_suffix(' >') }
+      def get_breadcrumbs
+        css('.pytorch-breadcrumbs > li').map {
+          |node| node.content.delete_suffix(' >').strip
+        }.reject { |item| item.nil? || item.empty? }
       end
 
       def get_name
-        # The id of the container `div.section` indicates the page type.
-        # If the id starts with `module-`, then it's an API reference,
-        # otherwise it is a note or design doc.
-        section_id = at_css('.section[id], section[id]')['id']
-        if section_id.starts_with? 'module-'
-          section_id.remove('module-')
-        else
-          name = get_breadcrumbs()[1]
-          NAME_REPLACEMENTS.fetch(name, name)
-        end
+        b = get_breadcrumbs
+        b[(b[1] == 'torch' ? 2 : 1)..].join('.')
       end
 
       def get_type
-        name
+        t = get_breadcrumbs[1]
+        TYPE_REPLACEMENTS.fetch(t, t)
       end
 
       def include_default_entry?
-        # Only include API references, and ignore notes or design docs
-        !subpath.start_with? 'generated/' and type.start_with? 'torch'
+        # Only include API entries to simplify and unify the list
+        return name.start_with?('torch.')
       end
 
       def additional_entries
         return [] if root_page?
 
         entries = []
-
-        css('dt').each do |node|
-          name = node['id']
-          if name == self.name or name == nil
+        css('dl').each do |node|
+          dt = node.at_css('dt')
+          if dt == nil
+            next
+          end
+          id = dt['id']
+          if id == name or id == nil
             next
           end
 
-          case node.parent['class']
-          when 'method', 'function'
-            if node.at_css('code').content.starts_with? 'property '
-              # this instance method is a property, so treat it as an attribute
-              entries << [name, node['id']]
-            else
-              entries << [name + '()', node['id']]
-            end
-          when 'class', 'attribute'
-            entries << [name, node['id']]
+          case node['class']
+          when 'py method', 'py function'
+            entries << [id + '()', id]
+          when 'py class', 'py attribute', 'py property'
+            entries << [id, id]
+          when 'footnote brackets', 'field-list simple'
+            next
           end
         end
 

lib/docs/scrapers/pytorch.rb

@@ -3,7 +3,6 @@ module Docs
     self.name = 'PyTorch'
     self.slug = 'pytorch'
     self.type = 'sphinx'
-    self.force_gzip = true
     self.links = {
       home: 'https://pytorch.org/',
       code: 'https://github.com/pytorch/pytorch'
@@ -11,21 +10,20 @@ module Docs
 
     html_filters.push 'pytorch/entries', 'pytorch/clean_html', 'sphinx/clean_html'
 
-    options[:skip] = ['cpp_index.html', 'packages.html', 'py-modindex.html', 'genindex.html']
+    options[:skip] = ['cpp_index.html', 'deploy.html', 'packages.html', 'py-modindex.html', 'genindex.html']
     options[:skip_patterns] = [/\Acommunity/, /\A_modules/, /\Anotes/, /\Aorg\/pytorch\//]
-    options[:max_image_size] = 256_000
 
     options[:attribution] = <<-HTML
-    &copy; 2019-2024 Torch Contributors<br>
-    Licensed under the 3-clause BSD License.
+    &copy; 2024, PyTorch Contributors<br>
+    PyTorch has a BSD-style license, as found in the <a href="https://github.com/pytorch/pytorch/blob/main/LICENSE">LICENSE</a> file.
     HTML
 
-    version '2' do
+    version '2.1' do
       self.release = '2.1'
       self.base_url = "https://pytorch.org/docs/#{release}/"
     end
 
-    version '1' do
+    version '1.13' do
       self.release = '1.13'
       self.base_url = "https://pytorch.org/docs/#{release}/"
     end