"""Base class for SciUnit scores."""
import imp
import logging
import math
import sys
from copy import copy
from typing import Tuple, Union
import numpy as np
from quantities import Quantity
from sciunit.base import SciUnit, config, ipy, log
from sciunit.errors import InvalidScoreError
# Set up score logger
score_logger = logging.getLogger("sciunit_scores")
if ipy:
imp.reload(logging)
sl_handler = logging.StreamHandler(sys.stdout)
score_logger.addHandler(sl_handler)
score_log_level = config.get("score_log_level", 1)
score_logger.setLevel(score_log_level)
[docs]class Score(SciUnit):
"""Abstract base class for scores."""
[docs] def __init__(
self, score: Union["Score", float, int, Quantity], related_data: dict = None
):
"""Abstract base class for scores.
Args:
score (Union['Score', float, int, Quantity], bool): A raw value to wrap in a Score class.
related_data (dict, optional): Artifacts to store with the score.
"""
self.check_score(score)
if related_data is None:
related_data = {}
self.score, self.related_data = score, related_data
if isinstance(score, Exception):
# Set to error score to use its summarize().
self.__class__ = ErrorScore
super(Score, self).__init__()
score = None
"""The score itself."""
_best = None
"""The best possible score of this type"""
_worst = None
"""The best possible score of this type"""
_allowed_types = None
"""List of allowed types for the score argument"""
_allowed_types_message = (
"Score of type %s is not an instance " "of one of the allowed types: %s"
)
"""Error message when score argument is not one of these types"""
_description = ""
"""A description of this score, i.e. how to interpret it.
Provided in the score definition"""
description = ""
"""A description of this score, i.e. how to interpret it.
For the user to set in bind_score"""
_raw = None
"""A raw number arising in a test's compute_score,
used to determine this score. Can be set for reporting a raw value
determined in Test.compute_score before any transformation,
e.g. by a Converter"""
related_data = None
"""Data specific to the result of a test run on a model."""
test = None
"""The test taken. Set automatically by Test.judge."""
model = None
"""The model judged. Set automatically by Test.judge."""
observation_schema = None
state_hide = ["related_data"]
[docs] @classmethod
def observation_preprocess(cls, observation: dict) -> dict:
return observation
[docs] @classmethod
def observation_postprocess(cls, observation: dict) -> dict:
return observation
[docs] def check_score(self, score: "Score") -> None:
"""Check the score with imposed additional constraints in the subclass on the score, e.g. the range of the allowed score.
Args:
score (Score): A sciunit score instance.
Raises:
InvalidScoreError: Exception raised if `score` is not a instance of sciunit score.
"""
if self._allowed_types and not isinstance(
score, self._allowed_types + (Exception,)
):
raise InvalidScoreError(
self._allowed_types_message % (type(score), self._allowed_types)
)
self._check_score(score)
[docs] def _check_score(self, score: "Score") -> None:
"""A method for each Score subclass to impose additional constraints on the score, e.g. the range of the allowed score.
Args:
score (Score): A sciunit score instance.
"""
[docs] @classmethod
def compute(cls, observation: dict, prediction: dict):
"""Compute whether the observation equals the prediction.
Args:
observation (dict): The observation from the real world.
prediction (dict): The prediction generated by a model.
Returns:
NotImplementedError: Not implemented error.
"""
return NotImplementedError("")
@property
def norm_score(self) -> "Score":
"""A floating point version of the score used for sorting.
If normalized = True, this must be in the range 0.0 to 1.0,
where larger is better (used for sorting and coloring tables).
Returns:
Score: The [0-1] normalized score.
"""
return self.score
@property
def log_norm_score(self) -> float:
"""The natural logarithm of the `norm_score`.
This is useful for guaranteeing convexity in an error surface.
Returns:
float: The natural logarithm of the `norm_score`.
"""
return math.log(self.norm_score) if self.norm_score is not None else None
@property
def log2_norm_score(self) -> float:
"""The logarithm base 2 of the `norm_score`.
This is useful for guaranteeing convexity in an error surface.
Returns:
float: The logarithm base 2 of the `norm_score`.
"""
return math.log2(self.norm_score) if self.norm_score is not None else None
@property
def log10_norm_score(self) -> float:
"""The logarithm base 10 of the `norm_score`.
This is useful for guaranteeing convexity in an error surface.
Returns:
float: The logarithm base 10 of the `norm_score`.
"""
return math.log10(self.norm_score) if self.norm_score is not None else None
[docs] def color(self, value: Union[float, "Score"] = None) -> tuple:
"""Turn the score into an RGB color tuple of three 8-bit integers.
Args:
value (Union[float,, optional): The score that will be turned to an RGB color. Defaults to None.
Returns:
tuple: A tuple of three 8-bit integers that represents an RGB color.
"""
if value is None:
value = self.norm_score
rgb = Score.value_color(value)
return rgb
[docs] @classmethod
def value_color(cls, value: Union[float, "Score"]) -> tuple:
"""Get a RGB color based on the Score.
Args:
value (Union[float,): [description]
Returns:
tuple: [description]
"""
import matplotlib.cm as cm
if value is None or np.isnan(value):
rgb = (128, 128, 128)
else:
cmap_low = config.get("cmap_low", 38)
cmap_high = config.get("cmap_high", 218)
cmap_range = cmap_high - cmap_low
cmap = cm.RdYlGn(int(cmap_range * value + cmap_low))[:3]
rgb = tuple([int(x * 256) for x in cmap])
return rgb
@property
def summary(self) -> str:
"""Summarize the performance of a model on a test.
Returns:
str: The summary of this score.
"""
return "=== Model %s achieved score %s on test '%s'. ===" % (
str(self.model),
str(self),
self.test,
)
[docs] def summarize(self):
"""[summary]"""
if self.score is not None:
log("%s" % self.summary)
[docs] def _describe(self) -> str:
"""Get the description of this score.
Returns:
str: The description of this score.
"""
result = "No description available"
if self.score is not None:
if self.description:
result = "%s" % self.description
elif self.test.score_type.__doc__:
result = self.describe_from_docstring()
return result
[docs] def describe_from_docstring(self) -> str:
"""Get the description of this score from the docstring.
Returns:
str: The description of this score.
"""
s = [self.test.score_type.__doc__.strip().replace("\n", "").replace(" ", "")]
if self.test.converter:
s += [self.test.converter.description]
s += [self._description]
result = "\n".join(s)
return result
[docs] def describe(self, quiet: bool = False) -> Union[str, None]:
"""Get the description of this score instance.
Args:
quiet (bool, optional): If `True`, then log the description, return the description otherwise.
Defaults to False.
Returns:
Union[str, None]: If not `quiet`, then return the description of this score instance.
Otherwise, `None`.
"""
d = self._describe()
return d
@property
def raw(self) -> str:
"""The raw score in string type.
Returns:
str: The raw score.
"""
value = self._raw if self._raw else self.score
if isinstance(value, (float, np.ndarray)):
string = "%.4g" % value
if hasattr(value, "magnitude"):
string += " %s" % str(value.units)[4:]
else:
string = None
return string
[docs] def get_raw(self) -> float:
"""Get the raw score. If there is not raw score, then get score.
Returns:
float: The raw score.
"""
value = copy(self._raw) if self._raw else copy(self.score)
return value
[docs] def set_raw(self, raw: float) -> None:
"""Set the raw score.
Args:
raw (float): The raw score to be set.
"""
self._raw = raw
[docs] def __repr__(self) -> str:
"""[summary]"""
return self.__str__()
[docs] def __str__(self) -> str:
"""[summary]"""
return "%s" % self.score
[docs] def __eq__(self, other: Union["Score", float]) -> bool:
"""[summary]"""
if isinstance(other, Score):
result = self.norm_score == other.norm_score
else:
result = self.score == other
return result
[docs] def __ne__(self, other: Union["Score", float]) -> bool:
"""[summary]"""
if isinstance(other, Score):
result = self.norm_score != other.norm_score
else:
result = self.score != other
return result
[docs] def __gt__(self, other: Union["Score", float]) -> bool:
"""[summary]"""
if isinstance(other, Score):
result = self.norm_score > other.norm_score
else:
result = self.score > other
return result
[docs] def __ge__(self, other: Union["Score", float]) -> bool:
"""[summary]"""
if isinstance(other, Score):
result = self.norm_score >= other.norm_score
else:
result = self.score >= other
return result
[docs] def __lt__(self, other: Union["Score", float]) -> bool:
"""[summary]"""
if isinstance(other, Score):
result = self.norm_score < other.norm_score
else:
result = self.score < other
return result
[docs] def __le__(self, other: Union["Score", float]) -> bool:
"""[summary]"""
if isinstance(other, Score):
result = self.norm_score <= other.norm_score
else:
result = self.score <= other
return result
[docs] def render_beautiful_msg(self, color: tuple, bg_brightness: int, msg: str):
result = ""
result += f"\x1b[38;2;{color[0]};{color[1]};{color[2]}m"
result += f"\x1b[48;2;{bg_brightness};{bg_brightness};{bg_brightness}m"
result += msg
result += "\x1b[0m"
return result
[docs] def log(self, **kwargs):
if self.norm_score is not None:
level = 100 - math.floor(self.norm_score * 99)
else:
level = 50
kwargs = {
k: v
for k, v in kwargs.items()
if k in ["exc_info", "stack_info", "stacklevel", "extra"]
}
msg = "Score: %s for %s on %s" % (self, self.model, self.test)
color = self.color()
bg_brightness = config.get("score_bg_brightness", 50)
msg = self.render_beautiful_msg(color, bg_brightness, msg)
score_logger.log(level, msg, **kwargs)
@property
def score_type(self):
"""The type of the score.
Returns:
str: the name of the score class.
"""
return self.__class__.__name__
[docs]class ErrorScore(Score):
"""A score returned when an error occurs during testing."""
@property
def norm_score(self) -> float:
"""Get the norm score, which is 0.0 for `ErrorScore` instance.
Returns:
float: The norm score.
"""
return 0.0
@property
def summary(self) -> str:
"""Summarize the performance of a model on a test.
Returns:
str: A textual summary of the score.
"""
return "== Model %s did not complete test %s due to error '%s'. ==" % (
str(self.model),
str(self.test),
str(self.score),
)
[docs] def _describe(self) -> str:
return self.summary
[docs] def __str__(self) -> str:
return "Error"