diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e7aca7..d19b12f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Change Log +- 4.1.0 + - Added `softplus()` and `erf()` special functions + - Added reciprocal sqrt `rsqrt()` operation to unary interface + - Added `sinh()` and `cosh()` trigonometric functions + - 4.0.0 - Added `fromArray()` factory method to build from PHP array - `build()` and `quick()` factory methods are now deprecated diff --git a/benchmarks/Functions/SigmoidMatrixBench.php b/benchmarks/Functions/SigmoidMatrixBench.php new file mode 100644 index 0000000..a100906 --- /dev/null +++ b/benchmarks/Functions/SigmoidMatrixBench.php @@ -0,0 +1,45 @@ +a = Matrix::uniform(500, 500); + } + + /** + * @Subject + * @Iterations(5) + * @OutputTimeUnit("seconds", precision=3) + */ + public function chained() : void + { + $this->a->negate() + ->exp() + ->addScalar(1.0) + ->reciprocal(); + } + + /** + * @Subject + * @Iterations(5) + * @OutputTimeUnit("seconds", precision=3) + */ + public function map() : void + { + $this->a->sigmoid(); + } +} diff --git a/composer.json b/composer.json index 0a9a766..62a8464 100644 --- a/composer.json +++ b/composer.json @@ -38,7 +38,8 @@ "Tensor\\": "src/" }, "files": [ - "src/constants.php" + "src/constants.php", + "src/math.php" ] }, "autoload-dev": { diff --git a/docs/Matrix.md b/docs/Matrix.md index 9991149..7815983 100644 --- a/docs/Matrix.md +++ b/docs/Matrix.md @@ -414,6 +414,7 @@ See [Unary](interfaces/unary.md). Each method returns a new `Matrix`. - `square() : self` — square of the matrix element-wise - `sqrt() : self` — square root of the matrix - `reciprocal() : self` — element-wise reciprocal of the matrix +- `rsqrt() : self` — element-wise reciprocal square root of the matrix, i.e. `1 / sqrt(x)` - `exp() : self` — exponential of the matrix - `expm1() : self` — exponential of each element minus 1 - `log(float $base = M_E) : self` — logarithm of the matrix in the specified base @@ -423,7 +424,6 @@ See [Unary](interfaces/unary.md). Each method returns a new `Matrix`. - `ceil() : self` — round up to the nearest integer - `sign() : self` — element-wise sign indication (`1.0`, `-1.0`, `0.0`) - `negate() : self` — take the negative of each value element-wise -- `sigmoid() : self` — element-wise sigmoid of the matrix ## Trigonometric @@ -435,6 +435,9 @@ See [Trigonometric](interfaces/trigonometric.md). Each method returns a new `Mat - `acos()` — arc cosine of the matrix - `tan()` — tangent of the matrix - `atan()` — arc tangent of the matrix +- `tanh()` — hyperbolic tangent of the matrix +- `cosh()` — hyperbolic cosine of the matrix +- `sinh()` — hyperbolic sine of the matrix - `rad2deg()` — convert angles from radians to degrees - `deg2rad()` — convert angles from degrees to radians @@ -456,7 +459,10 @@ See [Reductions](interfaces/reductions.md). For a `Matrix`, row-wise reductions See [Special](interfaces/special.md). +- `sigmoid() : self` — the element-wise sigmoid of the matrix, i.e. `1 / (1 + exp(-x))`; always in `(0, 1)` +- `softplus() : self` — the element-wise softplus of the matrix: `log(1 + e^{x})`; a smooth approximation of `max(0, x)` with derivative equal to the sigmoid - `softmax() : self` — the softmax of the matrix; each row is normalized independently so the elements of every row sum to `1.0` and the shape is preserved +- `erf() : self` — the element-wise error function of the matrix; `erf(-x) = -erf(x)` and `erf(±5.0) ≈ ±1.0` ## Clipping diff --git a/docs/Vector.md b/docs/Vector.md index c4a3fd0..8b63421 100644 --- a/docs/Vector.md +++ b/docs/Vector.md @@ -314,6 +314,7 @@ See [Unary](interfaces/unary.md). Each method returns a new `Vector`. - `square() : self` — square the vector - `sqrt() : self` — square root of the vector - `reciprocal() : self` — element-wise reciprocal of the vector +- `rsqrt() : self` — element-wise reciprocal square root of the vector, i.e. `1 / sqrt(x)` - `exp() : self` — exponentiate each element - `expm1() : self` — exponential of each element minus 1 - `log(float $base = M_E) : self` — log to the given base of each element @@ -323,7 +324,6 @@ See [Unary](interfaces/unary.md). Each method returns a new `Vector`. - `ceil() : self` — round up to the nearest integer - `sign() : self` — element-wise sign indication (`1.0`, `-1.0`, `0.0`) - `negate() : self` — negate each value -- `sigmoid() : self` — element-wise sigmoid of the vector ## Trigonometric @@ -336,6 +336,8 @@ See [Trigonometric](interfaces/trigonometric.md). Each method returns a new `Vec - `tan()` — tangent of the vector - `atan()` — arc tangent of the vector - `tanh()` — hyperbolic tangent of the vector +- `cosh()` — hyperbolic cosine of the vector +- `sinh()` — hyperbolic sine of the vector - `rad2deg()` — convert angles from radians to degrees - `deg2rad()` — convert angles from degrees to radians @@ -356,7 +358,10 @@ See [Reductions](interfaces/reductions.md). For a `Vector` these return scalar ` See [Special](interfaces/special.md). +- `sigmoid() : self` — the element-wise sigmoid of the vector, i.e. `1 / (1 + exp(-x))`; always in `(0, 1)` +- `softplus() : self` — the element-wise softplus of the vector: `log(1 + e^{x})`; a smooth approximation of `max(0, x)` with derivative equal to the sigmoid - `softmax() : self` — the softmax of the vector; the whole vector is treated as a single row so the elements sum to `1.0` +- `erf() : self` — the element-wise error function of the vector; `erf(-x) = -erf(x)` and `erf(±5.0) ≈ ±1.0` ## Clipping diff --git a/docs/interfaces/special.md b/docs/interfaces/special.md index 2ae1013..9010bf4 100644 --- a/docs/interfaces/special.md +++ b/docs/interfaces/special.md @@ -6,7 +6,7 @@ Special tensor functions. ## Overview -`Special` defines higher-level functions that combine a transform with a normalization across more than one element. +`Special` defines higher-order transcendental functions — element-wise transforms (`sigmoid`, `softplus`) and row-wise normalizations (`softmax`). ```php interface Special @@ -14,6 +14,18 @@ interface Special ## Methods +### `sigmoid() : mixed` + +Return the element-wise sigmoid of the tensor, i.e. `1 / (1 + exp(-x))`. + +The result is always in the open interval `(0, 1)` and saturates to `1.0` and `0.0` for large positive and negative inputs respectively. + +### `softplus() : mixed` + +Return the element-wise softplus of the tensor: `log(1 + exp(x))`. + +A smooth approximation of `max(0, x)` whose derivative is the sigmoid. Computed as `x < 0: log(1 + exp(x))` and `x >= 0: x + log(1 + exp(-x))` so the exponentially large argument never overflows. + ### `softmax() : mixed` Return the softmax of the tensor. @@ -23,3 +35,9 @@ The maximum of each normalized row is subtracted before exponentiating, so the r - For `Matrix` each row is normalized independently, so the elements of every row sum to `1.0` and the shape is preserved. Normalization is across the columns *within* each row, never across rows. - For `Vector` the whole vector is a single row, so its elements sum to `1.0`. - For `ColumnVector` the whole column is a single row, so its elements sum to `1.0`. + +### `erf() : mixed` + +Return the element-wise error function of the tensor: `erf(x) = (2 / sqrt(pi)) * integral_0^x exp(-t^2) dt`. + +The result is in the range `[-1.0, 1.0]` and is an odd function: `erf(-x) = -erf(x)`. For `|x| >= 5.0` the value saturates to `sign(x)`. diff --git a/docs/interfaces/tensor.md b/docs/interfaces/tensor.md index 57f2a19..210fccd 100644 --- a/docs/interfaces/tensor.md +++ b/docs/interfaces/tensor.md @@ -20,10 +20,10 @@ interface Tensor extends ArrayLike, Arithmetic, Comparable, | [ArrayLike](arraylike.md) | `shape`, `shapeString`, `size`, `map`, `reduce`, `asArray` | | [Arithmetic](arithmetic.md) | `multiply`, `divide`, `add`, `subtract`, `pow`, `mod` | | [Comparable](comparable.md) | `equal`, `notEqual`, `greater`, `greaterEqual`, `less`, `lessEqual` | -| [Unary](unary.md) | `abs`, `square`, `sqrt`, `reciprocal`, `exp`, `expm1`, `log`, `log1p`, `round`, `floor`, `ceil`, `clip`, `clipLower`, `clipUpper`, `sign`, `negate`, `sigmoid` | -| [Trigonometric](trigonometric.md) | `sin`, `asin`, `cos`, `acos`, `tan`, `atan`, `tanh`, `rad2deg`, `deg2rad` | +| [Unary](unary.md) | `abs`, `square`, `sqrt`, `reciprocal`, `rsqrt`, `exp`, `expm1`, `log`, `log1p`, `round`, `floor`, `ceil`, `clip`, `clipLower`, `clipUpper`, `sign`, `negate` | +| [Trigonometric](trigonometric.md) | `sin`, `asin`, `cos`, `acos`, `tan`, `atan`, `tanh`, `cosh`, `sinh`, `rad2deg`, `deg2rad` | | [Reductions](reductions.md) | `sum`, `product`, `min`, `max`, `mean`, `variance`, `median`, `quantile` | -| [Special](special.md) | `softmax` | +| [Special](special.md) | `sigmoid`, `softplus`, `softmax`, `erf` | Additionally, because `ArrayLike` extends `ArrayAccess`, `IteratorAggregate`, and `Countable`, every tensor is array-accessible, iterable, and countable. diff --git a/docs/interfaces/trigonometric.md b/docs/interfaces/trigonometric.md index f224c43..fcc1add 100644 --- a/docs/interfaces/trigonometric.md +++ b/docs/interfaces/trigonometric.md @@ -1,12 +1,12 @@ # Trigonometric -Element-wise trigonometric functions. +Element-wise trigonometric and hyperbolic functions. - **Namespace:** `Tensor\Trigonometric` ## Overview -`Trigonometric` defines element-wise trigonometric operations. Each method returns a new tensor of the same shape as `$this`. Angles are expressed in radians unless converting to/from degrees. +`Trigonometric` defines element-wise trigonometric and hyperbolic operations. Each method returns a new tensor of the same shape as `$this`. Angles are expressed in radians unless converting to/from degrees. ```php interface Trigonometric @@ -40,7 +40,15 @@ Compute the arc tangent of the tensor. ### `tanh() : mixed` -Return the hyperbolic tangent of the tensor. +Return the hyperbolic tangent of the tensor: `tanh(x) = sinh(x) / cosh(x)`. + +### `cosh() : mixed` + +Return the hyperbolic cosine of the tensor: `cosh(x) = (e^x + e^(-x)) / 2`. + +### `sinh() : mixed` + +Return the hyperbolic sine of the tensor: `sinh(x) = (e^x - e^(-x)) / 2`. ### `rad2deg() : mixed` @@ -48,4 +56,4 @@ Convert angles from radians to degrees. ### `deg2rad() : mixed` -Convert angles from degrees to radians. \ No newline at end of file +Convert angles from degrees to radians. diff --git a/docs/interfaces/unary.md b/docs/interfaces/unary.md index 0ca89cb..44f9551 100644 --- a/docs/interfaces/unary.md +++ b/docs/interfaces/unary.md @@ -30,6 +30,10 @@ Return the square root of the tensor. Return the element-wise reciprocal of the tensor. +### `rsqrt() : mixed` + +Return the reciprocal square root of the tensor, i.e. `1 / sqrt(x)`. + ### `exp() : mixed` Return the exponential of the tensor. @@ -84,10 +88,4 @@ Return the element-wise sign indication (`1.0`, `-1.0`, or `0.0`). ### `negate() : mixed` -Negate the tensor, i.e. take the negative. - -### `sigmoid() : mixed` - -Return the element-wise sigmoid of the tensor, i.e. `1 / (1 + exp(-x))`. - -The result is always in the open interval `(0, 1)` and saturates to `1.0` and `0.0` for large positive and negative inputs respectively. \ No newline at end of file +Negate the tensor, i.e. take the negative. \ No newline at end of file diff --git a/src/Matrix.php b/src/Matrix.php index 608a3c0..52244e8 100644 --- a/src/Matrix.php +++ b/src/Matrix.php @@ -1409,6 +1409,16 @@ public function sqrt() : self return $this->map('sqrt'); } + /** + * Return the reciprocal square root of the matrix. + * + * @return self + */ + public function rsqrt() : self + { + return $this->map('\Tensor\rsqrt'); + } + /** * Return the exponential of the matrix. * @@ -1532,6 +1542,26 @@ public function tanh() : self return $this->map('tanh'); } + /** + * Return the hyperbolic cosine of the matrix. + * + * @return self + */ + public function cosh() : self + { + return $this->map('cosh'); + } + + /** + * Return the hyperbolic sine of the matrix. + * + * @return self + */ + public function sinh() : self + { + return $this->map('sinh'); + } + /** * Convert angles from radians to degrees. * @@ -1929,10 +1959,7 @@ public function negate() : self */ public function sigmoid() : self { - return $this->negate() - ->exp() - ->addScalar(1.0) - ->reciprocal(); + return $this->map('\Tensor\sigmoid'); } /** @@ -1943,12 +1970,55 @@ public function sigmoid() : self */ public function softmax() : self { - $a = $this->subtractColumnVector($this->max()) - ->exp(); + $a = $this->subtractColumnVector($this->max())->exp(); return $a->divideColumnVector($a->sum()); } + /** + * Return the element-wise error function of the matrix. + * + * @return self + */ + public function erf() : self + { + $b = []; + + foreach ($this->a as $rowA) { + $rowB = []; + + foreach ($rowA as $valueA) { + $rowB[] = erf($valueA); + } + + $b[] = $rowB; + } + + return self::fromArray($b, false); + } + + /** + * Return the element-wise softplus of the matrix. + * + * @return self + */ + public function softplus() : self + { + $b = []; + + foreach ($this->a as $rowA) { + $rowB = []; + + foreach ($rowA as $valueA) { + $rowB[] = softplus($valueA); + } + + $b[] = $rowB; + } + + return self::fromArray($b, false); + } + /** * Attach matrix b above this matrix. * diff --git a/src/Special.php b/src/Special.php index f39e7ff..5cb6447 100644 --- a/src/Special.php +++ b/src/Special.php @@ -4,6 +4,20 @@ interface Special { + /** + * Return the element-wise sigmoid of the tensor. + * + * @return mixed + */ + public function sigmoid(); + + /** + * Return the element-wise softplus of the tensor. + * + * @return mixed + */ + public function softplus(); + /** * Return the softmax of the tensor. * @@ -13,4 +27,11 @@ interface Special * @return mixed */ public function softmax(); + + /** + * Return the element-wise error function of the tensor. + * + * @return mixed + */ + public function erf(); } diff --git a/src/Trigonometric.php b/src/Trigonometric.php index db23282..123cd61 100644 --- a/src/Trigonometric.php +++ b/src/Trigonometric.php @@ -53,6 +53,20 @@ public function atan(); */ public function tanh(); + /** + * Return the hyperbolic cosine of the tensor. + * + * @return mixed + */ + public function cosh(); + + /** + * Return the hyperbolic sine of the tensor. + * + * @return mixed + */ + public function sinh(); + /** * Convert angles from radians to degrees. * diff --git a/src/Unary.php b/src/Unary.php index 410fa7a..7d4fa49 100644 --- a/src/Unary.php +++ b/src/Unary.php @@ -32,6 +32,13 @@ public function sqrt(); */ public function reciprocal(); + /** + * Return the reciprocal square root of the tensor. + * + * @return mixed + */ + public function rsqrt(); + /** * Return the exponential of the tensor. * @@ -121,11 +128,4 @@ public function sign(); * @return mixed */ public function negate(); - - /** - * Return the element-wise sigmoid of the tensor. - * - * @return mixed - */ - public function sigmoid(); } diff --git a/src/Vector.php b/src/Vector.php index 9e1c13e..5b40f7e 100644 --- a/src/Vector.php +++ b/src/Vector.php @@ -1014,6 +1014,16 @@ public function sqrt() : self return $this->map('sqrt'); } + /** + * Return the reciprocal square root of the vector. + * + * @return self + */ + public function rsqrt() : self + { + return $this->map('\Tensor\rsqrt'); + } + /** * Exponentiate each element in the vector. * @@ -1131,6 +1141,26 @@ public function tanh() : self return $this->map('tanh'); } + /** + * Return the hyperbolic cosine of this vector. + * + * @return self + */ + public function cosh() : self + { + return $this->map('cosh'); + } + + /** + * Return the hyperbolic sine of this vector. + * + * @return self + */ + public function sinh() : self + { + return $this->map('sinh'); + } + /** * Convert angles from radians to degrees. * @@ -1446,10 +1476,7 @@ public function negate() : self */ public function sigmoid() : self { - return $this->negate() - ->exp() - ->addScalar(1.0) - ->reciprocal(); + return $this->map('\Tensor\sigmoid'); } /** @@ -1466,6 +1493,38 @@ public function softmax() : self return $a->divideScalar($a->sum()); } + /** + * Return the element-wise error function of the vector. + * + * @return self + */ + public function erf() : self + { + $b = []; + + foreach ($this->a as $valueA) { + $b[] = erf($valueA); + } + + return static::fromArray($b, false); + } + + /** + * Return the element-wise softplus of the vector. + * + * @return self + */ + public function softplus() : self + { + $b = []; + + foreach ($this->a as $valueA) { + $b[] = softplus($valueA); + } + + return static::fromArray($b, false); + } + /** * Multiply this vector with a matrix. * diff --git a/src/math.php b/src/math.php new file mode 100644 index 0000000..f8865bf --- /dev/null +++ b/src/math.php @@ -0,0 +1,91 @@ += 0: x + log(1 + exp(-x)) + * + * @param float $x + * @return float + */ +function softplus(float $x) : float +{ + return $x < 0.0 ? log1p(exp($x)) : $x + log1p(exp(-$x)); +} + +/** + * Return the reciprocal square root of $x, defined as 1 / sqrt(x). + * + * @param float $x + * @return float + */ +function rsqrt(float $x) : float +{ + return 1.0 / sqrt($x); +} + +/** + * Return the sigmoid of $x, defined as 1 / (1 + exp(-x)). + * + * The result is always in the open interval (0, 1) and saturates to + * 1.0 and 0.0 for large positive and negative arguments respectively. + * + * @param float $x + * @return float + */ +function sigmoid(float $x) : float +{ + return 1.0 / (1.0 + exp(-$x)); +} + +/** + * Return the error function of $x. + * + * The error function is defined as + * erf(x) = (2/sqrt(pi)) * integral_0^x exp(-t^2) dt + * + * For |x| <= 5 the function is evaluated using its Taylor power series, + * which reaches a relative error below 1e-14. For |x| > 5 the function + * returns the sign of x saturated to +/-1.0, which is correct to + * better than 1e-11 relative error over the remaining domain. + * + * @param float $x + * @return float + */ +function erf(float $x) : float +{ + switch (true) { + case $x === 0.0: + return 0.0; + + case $x >= 5.0: + return 1.0; + + case $x <= -5.0: + return -1.0; + } + + $scale = 2.0 / M_SQRTPI; + + $sum = 0.0; + $term = $x; + + for ($n = 0;; ++$n) { + if ($n > 0) { + $term *= -(2 * $n - 1) * $x * $x / ($n * (2 * $n + 1)); + } + + $sum += $term; + + if (abs($term) < 1e-18 * max(1.0, abs($sum))) { + break; + } + } + + return $scale * $sum; +} diff --git a/tests/ColumnVectorTest.php b/tests/ColumnVectorTest.php index eec1d57..d15b977 100644 --- a/tests/ColumnVectorTest.php +++ b/tests/ColumnVectorTest.php @@ -463,6 +463,24 @@ public function sigmoidPreservesColumnVector() : void $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); } + #[Test] + public function rsqrtPreservesColumnVector() : void + { + $a = ColumnVector::fromArray([2.0, 4.0, 8.0], false); + + $b = $a->rsqrt(); + + $this->assertInstanceOf(ColumnVector::class, $b); + + $expected = ColumnVector::fromArray([ + 0.70710678118655, + 0.5, + 0.35355339059327, + ], false); + + $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); + } + #[Test] public function tanhPreservesColumnVector() : void { diff --git a/tests/MatrixTest.php b/tests/MatrixTest.php index 0a5e5bd..be2145b 100644 --- a/tests/MatrixTest.php +++ b/tests/MatrixTest.php @@ -2130,6 +2130,26 @@ public function sqrt() : void $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); } + #[Test] + public function rsqrt() : void + { + $a = Matrix::fromArray([ + [13.0], + [11.0], + [9.0], + ], false); + + $b = $a->rsqrt(); + + $expected = Matrix::fromArray([ + [0.27735009811261], + [0.30151134457776], + [0.33333333333333], + ], false); + + $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); + } + #[Test] public function exp() : void { @@ -2569,6 +2589,78 @@ public function tanh() : void $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); } + #[Test] + public function cosh() : void + { + $a = Matrix::fromArray([ + [-1.0, 0.0, 1.0], + [2.0, -2.0, 0.5], + ], false); + + $b = $a->cosh(); + + $expected = Matrix::fromArray([ + [1.5430806348152437, 1.0, 1.5430806348152437], + [3.7621956910836314, 3.7621956910836314, 1.1276259652063807], + ], false); + + $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); + } + + #[Test] + public function sinh() : void + { + $a = Matrix::fromArray([ + [-1.0, 0.0, 1.0], + [2.0, -2.0, 0.5], + ], false); + + $b = $a->sinh(); + + $expected = Matrix::fromArray([ + [-1.1752011936438014, 0.0, 1.1752011936438014], + [3.6268604078470186, -3.6268604078470186, 0.5210953054937474], + ], false); + + $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); + } + + #[Test] + public function erf() : void + { + $a = Matrix::fromArray([ + [-1.0, 0.0, 1.0], + [2.0, -2.0, 0.5], + ], false); + + $b = $a->erf(); + + $expected = Matrix::fromArray([ + [-0.8427007929497148, 0.0, 0.8427007929497148], + [0.9953222650189527, -0.9953222650189527, 0.5204998778130465], + ], false); + + $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); + } + + #[Test] + public function softplus() : void + { + $a = Matrix::fromArray([ + [-1.0, 0.0, 1.0], + [2.0, -2.0, 0.5], + ], false); + + $b = $a->softplus(); + + $expected = Matrix::fromArray([ + [0.31326168751822286, 0.6931471805599453, 1.3132616875182228], + [2.1269280110429727, 0.1269280110429727, 0.9740769841801067], + ], false); + + $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); + } + #[Test] public function softmax() : void { diff --git a/tests/VectorTest.php b/tests/VectorTest.php index 5940101..7dc12ea 100644 --- a/tests/VectorTest.php +++ b/tests/VectorTest.php @@ -1204,6 +1204,21 @@ public function sqrt() : void $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); } + #[Test] + public function rsqrt() : void + { + $a = Vector::fromArray([4.0, 6.5, 2.9, 20.0, 2.6, 11.9], false); + + $b = $a->rsqrt(); + + $expected = Vector::fromArray([ + 0.5, 0.39223227027637, 0.5872202195147, + 0.22360679774998, 0.62017367294604, 0.28988551782622, + ], false); + + $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); + } + #[Test] public function exp() : void { @@ -1469,6 +1484,106 @@ public function tanh() : void $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); } + #[Test] + public function cosh() : void + { + $a = Vector::fromArray([-1.0, -0.5, 0.0, 0.5, 1.0, 2.0], false); + + $b = $a->cosh(); + + $expected = Vector::fromArray([ + 1.5430806348152437, + 1.1276259652063807, + 1.0, + 1.1276259652063807, + 1.5430806348152437, + 3.7621956910836314, + ], false); + + $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); + } + + #[Test] + public function sinh() : void + { + $a = Vector::fromArray([-1.0, -0.5, 0.0, 0.5, 1.0, 2.0], false); + + $b = $a->sinh(); + + $expected = Vector::fromArray([ + -1.1752011936438014, + -0.5210953054937474, + 0.0, + 0.5210953054937474, + 1.1752011936438014, + 3.6268604078470186, + ], false); + + $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); + } + + #[Test] + public function erf() : void + { + $a = Vector::fromArray([-2.0, -1.0, -0.5, 0.0, 0.5, 1.0, 2.0], false); + + $b = $a->erf(); + + $expected = Vector::fromArray([ + -0.9953222650189527, + -0.8427007929497148, + -0.5204998778130465, + 0.0, + 0.5204998778130465, + 0.8427007929497148, + 0.9953222650189527, + ], false); + + $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); + } + + #[Test] + public function erfSaturatesForLargeMagnitudes() : void + { + $this->assertEqualsWithDelta(1.0, Vector::fromArray([5.0], false)->erf()->asArray()[0], self::MAX_DELTA); + $this->assertEqualsWithDelta(-1.0, Vector::fromArray([-5.0], false)->erf()->asArray()[0], self::MAX_DELTA); + $this->assertEqualsWithDelta(1.0, Vector::fromArray([100.0], false)->erf()->asArray()[0], self::MAX_DELTA); + $this->assertEqualsWithDelta(-1.0, Vector::fromArray([-100.0], false)->erf()->asArray()[0], self::MAX_DELTA); + } + + #[Test] + public function softplus() : void + { + $a = Vector::fromArray([-2.0, -1.0, -0.5, 0.0, 0.5, 1.0, 2.0], false); + + $b = $a->softplus(); + + $expected = Vector::fromArray([ + 0.1269280110429727, + 0.313261687518223, + 0.47407698418010675, + 0.6931471805599453, + 0.9740769841801067, + 1.3132616875182228, + 2.1269280110429727, + ], false); + + $this->assertEqualsWithDelta($expected, $b, self::MAX_DELTA); + } + + #[Test] + public function softplusIsCloseToIdentityForLargePositiveValues() : void + { + $a = Vector::fromArray([10.0, 20.0, 50.0], false); + + $b = $a->softplus(); + + // softplus(x) = log(1 + exp(x)) is very close to x for x >> 0 + $this->assertEqualsWithDelta(10.0, $b->asArray()[0], 1e-3); + $this->assertEqualsWithDelta(20.0, $b->asArray()[1], 1e-7); + $this->assertEqualsWithDelta(50.0, $b->asArray()[2], 1e-9); + } + #[Test] public function softmax() : void {