Coverage for pySDC/helpers/testing.py: 100%

1#!/usr/bin/env python3 

2# -*- coding: utf-8 -*- 

3""" 

4Created on Fri Feb 9 14:41:43 2024 

5 

6Helpers module for testing utilities 

7""" 

8 

9import os 

10import json 

11import warnings 

12import numpy as np 

13 

14 

15class DataChecker: 

16 """ 

17 Object allowing to quickly generate and check generated data from scripts. 

18 

19 Here is an example of use for one given script: 

20 

21 >>> from pySDC.helpers.testing import DataChecker 

22 >>> 

23 >>> data = DataChecker(__file__) 

24 >>> # ... some computations ... 

25 >>> data.storeAndCheck('solution', solution) 

26 >>> # ... some other computations ... 

27 >>> data.storeAndCheck('errors', errors) 

28 >>> # ... ploting some figures 

29 >>> data.writeToJSON() # end of script 

30 

31 The `storeAndCheck` method use a unique key as first argument, and the 

32 data as second argument, into list format. 

33 Calling this method will store the data into a cache variable, and compare 

34 with reference data stored in `_dataRef.json` (if it exists, see below ...). 

35 Finally, the `writeToJSON` method saves the cache variable into a json 

36 file `_data.json`. 

37 

38 When there is no `_dataRef.json` in the script directory, 

39 executing the script output some warning telling that there is no 

40 reference data to compare with. 

41 To remove those warnings (and properly test data), just rename the 

42 `_data.json` into `_dataRef.json`. 

43 Then re-running the script will then compare the newly generated data with 

44 the reference data stored into `_dataRef.json`, and raise an error if there 

45 is some differences. 

46 

47 Important 

48 --------- 

49 

50 - using the `__file__` built-in variable is necessary to save data in the 

51 script directory, independently of the working directory. 

52 - several script in the same directory can use the DataChecker, which 

53 implies that the key provided to `storeAndCheck` must be unique across 

54 all directory scripts. If not, the same key from previous directory will 

55 simply be overwritten in the cached variable and the `_data.ref` file. 

56 """ 

57 

58 def __init__(self, filePath): 

59 """ 

60 Parameters 

61 ---------- 

62 filePath : str 

63 Absolute path of the calling script, usually `__file__`. The data files are in its directory. 

64 """ 

65 path = '/' + os.path.join(*filePath.split('/')[:-1]) 

66 self._data = {} # cache for data 

67 self._dataRef = None # cache for reference data 

68 self._dataFile = os.path.join(path, '_data.json') 

69 self._dataRefFile = os.path.join(path, '_dataRef.json') 

70 

71 def storeAndCheck(self, key, data, rtol=1e-5, atol=1e-8): 

72 """ 

73 Store data into cache, and check with eventual reference data 

74 

75 Parameters 

76 ---------- 

77 key : str 

78 Unique key (project wide) for the data. 

79 data : list or array-like 

80 The data that has to be stored. 

81 rtol : float 

82 Relative tolerance 

83 atol : float 

84 Absolute tolerance 

85 """ 

86 self._data[key] = list(data) 

87 if self._dataRef is None: 

88 try: 

89 with open(self._dataRefFile, "r") as f: 

90 self._dataRef = json.load(f) 

91 except FileNotFoundError: 

92 warnings.warn(f"no reference data to check key:{key}") 

93 return # pragma: no cover 

94 

95 assert key in self._dataRef, f"key:{key} not in reference data" 

96 

97 data = self._data[key] 

98 ref = self._dataRef[key] 

99 

100 assert len(data) == len(ref), f"data with len:{len(data)}) cannot be compared to ref with len:{len(ref)})" 

101 

102 # A reference stored as inf is a run that crashed. Those are not compared: at the edge of stability, whether 

103 # a solver gives up or just converges is decided by round-off, so a different summation order or platform 

104 # can turn it into a finite value. A finite reference that now crashes still fails. 

105 compared = ~np.isinf(np.asarray(ref, dtype=float)) 

106 assert np.allclose( 

107 np.asarray(data, dtype=float)[compared], 

108 np.asarray(ref, dtype=float)[compared], 

109 atol=atol, 

110 rtol=rtol, 

111 equal_nan=True, 

112 ), f"{key}, difference between data:{data} and ref:{ref}" 

113 

114 def writeToJSON(self): 

115 """Write cached data into a json file""" 

116 if os.path.isfile(self._dataFile): 

117 with open(self._dataFile, "r") as f: 

118 self._data.update(json.load(f)) 

119 with open(self._dataFile, "w") as f: 

120 json.dump(self._data, f)