Why Does HuggingFaceEndpoint Skip Quantization and Tokenizer Configuration Unlike HuggingFacePipeline?

I want to build a base chain class to instantiate chains with an LLM of my choice (e.g. gpt-4o-mini or meta-llama/Meta-Llama-3-8B).

While I noticed that HuggingFacePipeline runs locally and requires explicit configuration (like setting up a tokenizer and quantization via BitsAndBytesConfig), I found an alternative in HuggingFaceEndpoint (formerly HuggingFaceHub).

This raises the question: why does HuggingFaceEndpoint not need a quantization config or a tokenizer?

It seems that remote endpoints handle model loading and configuration (e.g. quantization and tokenization) on the server side, whereas local pipelines require these details to be provided manually. I'm looking for clarification or an example guide on how the endpoint abstracts these configurations so I can understand the requirements for using these open source LLMs in production.

Below is a pseudocode snippet illustrating the approach I had in mind (note: this code is for conceptual understanding and might not run as-is):

PYTHONCopied
class BaseAgentChain:

    def __init__(self, model_name="gpt-4o-mini", temperature=0, system_prompt_shell=None, user_prompt_shell=None, tools=None, **agent_kwargs):
        self.model_name = model_name
        self.temperature = temperature
        self.tools = tools if tools is not None else []

        self.system_prompt = system_prompt_shell
        self.user_prompt = user_prompt_shell
        self.memory = ConversationBufferMemory()

        self.agent_kwargs = agent_kwargs
        self.task = agent_kwargs.get('task', None)
        self.quant = agent_kwargs.get('quant_config', None)

        # Initialize the LLM model
        try:
            print(f"Initializing BaseAgent with model: {model_name} and temperature: {temperature}")
            self.llm = self._set_llm()
            print("Model initialized successfully.")
        except Exception as e:
            print(f"Error initializing model: {e}")
            raise RuntimeError(f"Failed to initialize the model: {e}")

    def _set_llm(self):
        try:
            available_llm_apis = os.getenv("AVAILABLE_LLM_APIS", "openai").split(',')
            print(f"Current available LLM APIs as LLM sources to BaseAgentChain: {available_llm_apis}")

            source_by = os.getenv("LLM_SOURCE", "openai")
            print(f"Initializing model `{self.model_name}` with `{source_by}`")

            if source_by == 'openai':
                llm = ChatOpenAI(
                    model=self.model_name,
                    temperature=self.temperature,
                    openai_api_key=os.getenv('OPENAI_API_KEY')
                )
                return llm

            elif source_by == 'hugging_face':
                from transformers import (
                    AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, pipeline
                )

                HF_TOKEN = os.getenv('HF_TOKEN')

                tokenizer = AutoTokenizer.from_pretrained(
                    self.model_name,
                    use_auth_token=HF_TOKEN
                )

                quant_config = BitsAndBytesConfig(**self.quant) if self.quant else None

                model = AutoModelForCausalLM.from_pretrained(
                    self.model_name,
                    device_map="auto",
                    quantization_config=quant_config,
                    use_auth_token=HF_TOKEN
                )

                pipe = pipeline(
                    self.task,
                    model=model,
                    tokenizer=tokenizer,
                    max_new_tokens=128
                )

                llm = HuggingFacePipeline(pipeline=pipe)
                return llm

            else:
                raise ValueError(f"Unsupported LLM source: {source_by}")

        except Exception as e:
            raise RuntimeError(f"LLM was not defined. Error: {e}")

I would appreciate any insights or documentation that explains the reasoning behind these configuration differences and how the remote endpoint manages them.