"""Unit tests for scores and score collections"""
import unittest
import numpy as np
from IPython.display import display
from pandas import DataFrame
from pandas.core.frame import DataFrame
from pandas.core.series import Series
from quantities import Quantity
from sciunit import Score, ScoreArray, ScoreMatrix
from sciunit.errors import InvalidScoreError
from sciunit.models import Model
from sciunit.scores import (
BooleanScore,
CohenDScore,
ErrorScore,
FloatScore,
InsufficientDataScore,
NAScore,
NoneScore,
PercentScore,
RandomScore,
RatioScore,
TBDScore,
ZScore,
)
from sciunit.scores.collections_m2m import ScoreArrayM2M, ScoreMatrixM2M
from sciunit.tests import RangeTest, Test
from sciunit.unit_test.base import SuiteBase
from sciunit.utils import NotebookTools
[docs]class ScoresTestCase(SuiteBase, unittest.TestCase, NotebookTools):
path = "."
[docs] def test_score_matrix_constructor(self):
tests = [Test([1, 2, 3])]
models = [Model()]
scores = np.array([ZScore(1.0)])
ScoreArray(tests)
scoreMatrix = ScoreMatrix(tests, models, scores)
scoreMatrix = ScoreMatrix(tests, models, scores, transpose=True)
tests = Test([1, 2, 3])
models = Model()
ScoreMatrix(tests, models, scores)
[docs] def test_score_matrix(self):
t, t1, t2, m1, m2 = self.prep_models_and_tests()
sm = t.judge(m1)
self.assertRaises(TypeError, sm.__getitem__, 0)
self.assertEqual(str(sm.get_group((t1, m1))), "Pass")
self.assertEqual(str(sm.get_group((m1, t1))), "Pass")
self.assertEqual(str(sm.get_group((m1.name, t1.name))), "Pass")
self.assertEqual(str(sm.get_group((t1.name, m1.name))), "Pass")
self.assertRaises(TypeError, sm.get_group, (0, 0))
self.assertRaises(KeyError, sm.get_by_name, "This name does not exist")
self.assertIsInstance(sm.__getattr__("score"), DataFrame)
self.assertIsInstance(sm.norm_scores, DataFrame)
self.assertIsInstance(sm.T, ScoreMatrix)
# self.assertIsInstance(sm.to_html(True, True, True), str)
# self.assertIsInstance(sm.to_html(), str)
self.assertTrue(type(sm) is ScoreMatrix)
self.assertTrue(sm[t1][m1].score)
self.assertTrue(sm["test1"][m1].score)
self.assertTrue(sm[m1]["test1"].score)
self.assertFalse(sm[t2][m1].score)
self.assertEqual(sm[(m1, t1)].score, True)
self.assertEqual(sm[(m1, t2)].score, False)
sm = t.judge([m1, m2])
self.assertEqual(sm.stature(t1, m1), 1)
self.assertEqual(sm.stature(t1, m2), 2)
display(sm)
######### m2m #################
t1.observation = [2, 3]
smm2m = ScoreMatrixM2M(
test=t1, models=[m1], scores=[[Score(1), Score(1)], [Score(1), Score(1)]]
)
self.assertIsInstance(smm2m.__getattr__("score"), DataFrame)
self.assertIsInstance(smm2m.__getattr__("norm_scores"), DataFrame)
self.assertIsInstance(smm2m.__getattr__("related_data"), DataFrame)
self.assertRaises(KeyError, smm2m.get_by_name, "Not Exist")
self.assertIsInstance(smm2m.norm_scores, DataFrame)
self.assertRaises(KeyError, smm2m.get_by_name, "Not Exist")
self.assertRaises(TypeError, smm2m.get_group, [0])
self.assertIsInstance(smm2m.get_group([m1.name, t1.name]), Score)
self.assertEqual(smm2m.get_group([m1.name, t1.name]).score, 1)
self.assertIsInstance(smm2m.get_group([m1, t1]), Score)
self.assertEqual(smm2m.get_group([m1, t1]).score, 1)
[docs] def test_score_arrays(self):
t, t1, t2, m1, m2 = self.prep_models_and_tests()
sm = t.judge(m1)
sa = sm[m1]
self.assertTrue(type(sa) is ScoreArray)
self.assertIsInstance(sa.__getattr__("score"), Series)
self.assertRaises(KeyError, sa.get_by_name, "This name does not exist")
self.assertEqual(list(sa.norm_scores.values), [1.0, 0.0])
self.assertEqual(sa.stature(t1), 1)
self.assertEqual(sa.stature(t2), 2)
self.assertEqual(sa.stature(t1), 1)
display(sa)
######### m2m #################
sam2m = ScoreArrayM2M(
test=t1, models=[m1], scores=[[Score(1), Score(1)], [Score(1), Score(1)]]
)
self.assertRaises(KeyError, sam2m.get_by_name, "Not Exist")
[docs] def test_regular_score_types_1(self):
self.assertEqual(PercentScore(0).norm_score, 0)
self.assertEqual(PercentScore(100).norm_score, 1)
score = PercentScore(42)
self.assertRaises(InvalidScoreError, PercentScore, 101)
self.assertRaises(InvalidScoreError, PercentScore, -1)
self.assertEqual(str(score), "42.0%")
self.assertEqual(score.norm_score, 0.42)
self.assertEqual(1, ZScore(0.0).norm_score)
self.assertEqual(0, ZScore(1e12).norm_score)
self.assertEqual(0, ZScore(-1e12).norm_score)
ZScore(0.7)
score = ZScore.compute({"mean": 3.0, "std": 1.0}, {"value": 2.0})
self.assertIsInstance(
ZScore.compute({"mean": 3.0}, {"value": 2.0}), InsufficientDataScore
)
self.assertIsInstance(
ZScore.compute({"mean": 3.0, "std": -1.0}, {"value": 2.0}),
InsufficientDataScore,
)
self.assertIsInstance(
ZScore.compute({"mean": np.nan, "std": np.nan}, {"value": np.nan}),
InsufficientDataScore,
)
self.assertEqual(score.score, -1.0)
self.assertEqual(1, CohenDScore(0.0).norm_score)
self.assertEqual(0, CohenDScore(1e12).norm_score)
self.assertEqual(0, CohenDScore(-1e12).norm_score)
CohenDScore(-0.3)
score = CohenDScore.compute(
{"mean": 3.0, "std": 1.0}, {"mean": 2.0, "std": 1.0}
)
self.assertAlmostEqual(-0.707, score.score, 3)
self.assertEqual("D = -0.71", str(score))
score = CohenDScore.compute(
{"mean": 3.0, "std": 10.0, "n": 10}, {"mean": 2.5, "std": 10.0, "n": 10}
)
self.assertAlmostEqual(-0.05, score.score, 2)
[docs] def test_regular_score_types_2(self):
BooleanScore(True)
BooleanScore(False)
score = BooleanScore.compute(5, 5)
self.assertEqual(score.norm_score, 1)
score = BooleanScore.compute(4, 5)
self.assertEqual(score.norm_score, 0)
self.assertEqual(1, BooleanScore(True).norm_score)
self.assertEqual(0, BooleanScore(False).norm_score)
t = RangeTest([2, 3])
score.test = t
score.describe()
score.description = "Lorem Ipsum"
score.describe()
score = FloatScore(3.14)
self.assertRaises(
InvalidScoreError, score.check_score, Quantity([1, 2, 3], "J")
)
obs = np.array([1.0, 2.0, 3.0])
pred = np.array([1.0, 2.0, 4.0])
score = FloatScore.compute_ssd(obs, pred)
self.assertEqual(str(score), "1")
self.assertEqual(score.score, 1.0)
score = RatioScore(1.2)
self.assertEqual(1, RatioScore(1.0).norm_score)
self.assertEqual(0, RatioScore(1e12).norm_score)
self.assertEqual(0, RatioScore(1e-12).norm_score)
self.assertEqual(str(score), "Ratio = 1.20")
self.assertRaises(InvalidScoreError, RatioScore, -1.0)
score = RatioScore.compute({"mean": 4.0, "std": 1.0}, {"value": 2.0})
self.assertEqual(score.score, 0.5)
[docs] def test_irregular_score_types(self):
e = Exception("This is an error")
score = ErrorScore(e)
score = NAScore(None)
score = TBDScore(None)
score = NoneScore(None)
score = NoneScore("this is a string")
self.assertIsInstance(str(score), str)
self.assertRaises(InvalidScoreError, NoneScore, ["this is a string list"])
score = InsufficientDataScore(None)
self.assertEqual(score.norm_score, None)
[docs] def test_only_lower_triangle(self):
"""Test validation of observations against the `observation_schema`."""
self.do_notebook("test_only_lower_triangle")
[docs] def test_RandomScore(self):
"""Note: RandomScore is only used for debugging purposes"""
score = RandomScore(0.5)
self.assertEqual("0.5", str(score))
[docs] def test_Score(self):
self.assertIsInstance(Score.compute({}, {}), NotImplementedError)
score = Score(0.5)
self.assertEqual(score.norm_score, 0.5)
self.assertAlmostEqual(score.log_norm_score, -0.693, 2)
self.assertAlmostEqual(score.log2_norm_score, -1.0, 1)
self.assertAlmostEqual(score.log10_norm_score, -0.301, 1)
self.assertIsInstance(score.raw, str)
score._raw = "this is a string"
self.assertIsNone(score.raw)
self.assertIsInstance(score.__repr__(), str)
self.assertIsInstance(score.__str__(), str)
self.assertFalse(score.__ne__(score))
self.assertTrue(score.__ne__(Score(998.0)))
self.assertFalse(score.__ne__(0.5))
self.assertTrue(score.__ne__(0.6))
self.assertFalse(score.__gt__(score))
self.assertTrue(score.__gt__(Score(0.2)))
self.assertFalse(score.__gt__(0.5))
self.assertTrue(score.__gt__(0.2))
self.assertFalse(score.__lt__(score))
self.assertTrue(score.__lt__(Score(0.9)))
self.assertFalse(score.__lt__(0.5))
self.assertTrue(score.__lt__(0.9))
self.assertTrue(score.__le__(score))
self.assertTrue(score.__le__(Score(0.5)))
self.assertTrue(score.__le__(0.5))
self.assertTrue(score.__le__(0.5))
self.assertFalse(score.__le__(0.1))
self.assertFalse(score.__le__(Score(0.1)))
self.assertIsInstance(score.score_type, str)
[docs] def test_ErrorScore(self):
score = ErrorScore(0.5)
self.assertEqual(0.0, score.norm_score)
self.assertIsInstance(score.summary, str)
self.assertIsInstance(score._describe(), str)
self.assertIsInstance(str(score), str)
if __name__ == "__main__":
unittest.main()