What is the correct, future-proof, way of deploying a pytorch python model in C++ for inference?

deployment
14

With this code :

import torch
from torch.export import export


# https://pytorch.org/docs/stable/export.html
class Mod(torch.nn.Module):
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        ner_results = nlp(text)
        return ner_results


seq_length = len(text)
batch_size = 1
example_args= (torch.rand(batch_size, seq_length))

#exported_program: torch.export.ExportedProgram = export(
    #Mod(), args=example_args
#)

ep = torch.export.export(Mod(), example_args)

print(exported_program)

I get this error :

import torch
from torch.export import export


# https://pytorch.org/docs/stable/export.html
class Mod(torch.nn.Module):
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        ner_results = nlp(text)
        return ner_results


seq_length = len(text)
batch_size = 1
example_args= (torch.rand(batch_size, seq_length))

#exported_program: torch.export.ExportedProgram = export(
    #Mod(), args=example_args
#)

ep = torch.export.export(Mod(), example_args)

print(exported_program)






    ep = torch.export.export(Mod(), example_args)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/raphy/BertFineTuningForNERPyTorch/.bftner/lib/python3.12/site-packages/torch/export/__init__.py", line 368, in export
    return _export(
           ^^^^^^^^
  File "/home/raphy/BertFineTuningForNERPyTorch/.bftner/lib/python3.12/site-packages/torch/export/_trace.py", line 1035, in wrapper
    raise e
  File "/home/raphy/BertFineTuningForNERPyTorch/.bftner/lib/python3.12/site-packages/torch/export/_trace.py", line 1008, in wrapper
    ep = fn(*args, **kwargs)
         ^^^^^^^^^^^^^^^^^^^
  File "/home/raphy/BertFineTuningForNERPyTorch/.bftner/lib/python3.12/site-packages/torch/export/exported_program.py", line 128, in wrapper
    return fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^
  File "/home/raphy/BertFineTuningForNERPyTorch/.bftner/lib/python3.12/site-packages/torch/export/_trace.py", line 1970, in _export
    return _export_for_training(
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/raphy/BertFineTuningForNERPyTorch/.bftner/lib/python3.12/site-packages/torch/export/_trace.py", line 1035, in wrapper
    raise e
  File "/home/raphy/BertFineTuningForNERPyTorch/.bftner/lib/python3.12/site-packages/torch/export/_trace.py", line 1008, in wrapper
    ep = fn(*args, **kwargs)
         ^^^^^^^^^^^^^^^^^^^
  File "/home/raphy/BertFineTuningForNERPyTorch/.bftner/lib/python3.12/site-packages/torch/export/exported_program.py", line 128, in wrapper
    return fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^
  File "/home/raphy/BertFineTuningForNERPyTorch/.bftner/lib/python3.12/site-packages/torch/export/_trace.py", line 1821, in _export_for_training
    ) = _process_export_inputs(mod, args, kwargs, dynamic_shapes)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/raphy/BertFineTuningForNERPyTorch/.bftner/lib/python3.12/site-packages/torch/export/_trace.py", line 1075, in _process_export_inputs
    raise UserError(
torch._dynamo.exc.UserError: Expecting `args` to be a tuple of example positional inputs, got <class 'torch.Tensor'>