timm/timm_image.py

# Copyright 2021 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import numpy
import os
from pathlib import Path

import towhee
from towhee.operator.base import NNOperator, OperatorFlag
from towhee.types.arg import arg, to_image_color
from towhee import register

import torch
from torch import nn

from PIL import Image as PILImage

import timm
from timm.data.transforms_factory import create_transform
from timm.data import resolve_data_config
from timm.models.factory import create_model

import warnings

warnings.filterwarnings('ignore')
log = logging.getLogger()


@register(output_schema=['vec'])
class TimmImage(NNOperator):
    """
    Pytorch image embedding operator that uses the Pytorch Image Model (timm) collection.
    Args:
        model_name (`str`):
            Which model to use for the embeddings.
        num_classes (`int = 1000`):
            Number of classes for classification.
        skip_preprocess (`bool = False`):
            Whether skip image transforms.
    """

    def __init__(self, model_name: str, num_classes: int = 1000, skip_preprocess: bool = False) -> None:
        super().__init__()
        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
        self.model_name = model_name
        self.model = create_model(self.model_name, pretrained=True, num_classes=num_classes)
        self.model.to(self.device)
        self.model.eval()
        self.config = resolve_data_config({}, model=self.model)
        self.tfms = create_transform(**self.config)
        self.skip_tfms = skip_preprocess

    @arg(1, to_image_color('RGB'))
    def __call__(self, img: towhee._types.Image) -> numpy.ndarray:
        img = PILImage.fromarray(img.astype('uint8'), 'RGB')
        if not self.skip_tfms:
            img = self.tfms(img).unsqueeze(0)
        img = img.to(self.device)
        features = self.model.forward_features(img)
        if features.dim() == 4:
            global_pool = nn.AdaptiveAvgPool2d(1)
            features = global_pool(features)

        features = features.to('cpu')
        vec = features.flatten().detach().numpy()
        return vec

    def save_model(self, format: str = 'pytorch', path: str = 'default'):
        if path == 'default':
            path = str(Path(__file__).parent)
            path = os.path.join(path, 'saved', format)
            os.makedirs(path, exist_ok=True)
            name = self.model_name.replace('/', '-')
            path = os.path.join(path, name)
        dummy_input = torch.rand((1,) + self.config['input_size'])
        if format == 'pytorch':
            path = path + '.pt'
            torch.save(self.model, path)
        elif format == 'torchscript':
            path = path + '.pt'
            try:
                try:
                    jit_model = torch.jit.script(self.model)
                except Exception:
                    jit_model = torch.jit.trace(self.model, dummy_input, strict=False)
                torch.jit.save(jit_model, path)
            except Exception as e:
                log.error(f'Fail to save as torchscript: {e}.')
                raise RuntimeError(f'Fail to save as torchscript: {e}.')
        elif format == 'onnx':
            path = path + '.onnx'
            try:
                torch.onnx.export(self.model,
                                  dummy_input,
                                  path,
                                  input_names=["input"],
                                  output_names=["output"],
                                  opset_version=12)
            except Exception as e:
                log.error(f'Fail to save as onnx: {e}.')
                raise RuntimeError(f'Fail to save as onnx: {e}.')
        # todo: elif format == 'tensorrt':
        else:
            log.error(f'Unsupported format "{format}".')

    @staticmethod
    def supported_model_names(format: str = None):
        full_list = timm.list_models(pretrained=True)
        full_list.sort()
        if format is None:
            model_list = full_list
        elif format == 'pytorch':
            to_remove = []
            assert set(to_remove).issubset(set(full_list))
            model_list = list(set(full_list) - set(to_remove))
        # todo: elif format == 'torchscript':
        # todo: elif format == 'tensorrt'
        else:
            log.error(f'Invalid format "{format}". Currently supported formats: "pytorch".')
        return model_list