ssim.c (17419B)
1 /* 2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #include <assert.h> 13 #include <math.h> 14 15 #include "config/aom_dsp_rtcd.h" 16 17 #include "aom_dsp/ssim.h" 18 #include "aom_ports/mem.h" 19 20 void aom_ssim_parms_8x8_c(const uint8_t *s, int sp, const uint8_t *r, int rp, 21 uint32_t *sum_s, uint32_t *sum_r, uint32_t *sum_sq_s, 22 uint32_t *sum_sq_r, uint32_t *sum_sxr) { 23 int i, j; 24 for (i = 0; i < 8; i++, s += sp, r += rp) { 25 for (j = 0; j < 8; j++) { 26 *sum_s += s[j]; 27 *sum_r += r[j]; 28 *sum_sq_s += s[j] * s[j]; 29 *sum_sq_r += r[j] * r[j]; 30 *sum_sxr += s[j] * r[j]; 31 } 32 } 33 } 34 35 static const int64_t cc1 = 26634; // (64^2*(.01*255)^2 36 static const int64_t cc2 = 239708; // (64^2*(.03*255)^2 37 static const int64_t cc1_10 = 428658; // (64^2*(.01*1023)^2 38 static const int64_t cc2_10 = 3857925; // (64^2*(.03*1023)^2 39 static const int64_t cc1_12 = 6868593; // (64^2*(.01*4095)^2 40 static const int64_t cc2_12 = 61817334; // (64^2*(.03*4095)^2 41 42 static double similarity(uint32_t sum_s, uint32_t sum_r, uint32_t sum_sq_s, 43 uint32_t sum_sq_r, uint32_t sum_sxr, int count, 44 uint32_t bd) { 45 double ssim_n, ssim_d; 46 int64_t c1 = 0, c2 = 0; 47 if (bd == 8) { 48 // scale the constants by number of pixels 49 c1 = (cc1 * count * count) >> 12; 50 c2 = (cc2 * count * count) >> 12; 51 } else if (bd == 10) { 52 c1 = (cc1_10 * count * count) >> 12; 53 c2 = (cc2_10 * count * count) >> 12; 54 } else if (bd == 12) { 55 c1 = (cc1_12 * count * count) >> 12; 56 c2 = (cc2_12 * count * count) >> 12; 57 } else { 58 assert(0); 59 // Return similarity as zero for unsupported bit-depth values. 60 return 0; 61 } 62 63 ssim_n = (2.0 * sum_s * sum_r + c1) * 64 (2.0 * count * sum_sxr - 2.0 * sum_s * sum_r + c2); 65 66 ssim_d = ((double)sum_s * sum_s + (double)sum_r * sum_r + c1) * 67 ((double)count * sum_sq_s - (double)sum_s * sum_s + 68 (double)count * sum_sq_r - (double)sum_r * sum_r + c2); 69 70 return ssim_n / ssim_d; 71 } 72 73 static double ssim_8x8(const uint8_t *s, int sp, const uint8_t *r, int rp) { 74 uint32_t sum_s = 0, sum_r = 0, sum_sq_s = 0, sum_sq_r = 0, sum_sxr = 0; 75 aom_ssim_parms_8x8(s, sp, r, rp, &sum_s, &sum_r, &sum_sq_s, &sum_sq_r, 76 &sum_sxr); 77 return similarity(sum_s, sum_r, sum_sq_s, sum_sq_r, sum_sxr, 64, 8); 78 } 79 80 // We are using a 8x8 moving window with starting location of each 8x8 window 81 // on the 4x4 pixel grid. Such arrangement allows the windows to overlap 82 // block boundaries to penalize blocking artifacts. 83 double aom_ssim2(const uint8_t *img1, const uint8_t *img2, int stride_img1, 84 int stride_img2, int width, int height) { 85 int i, j; 86 int samples = 0; 87 double ssim_total = 0; 88 89 // sample point start with each 4x4 location 90 for (i = 0; i <= height - 8; 91 i += 4, img1 += stride_img1 * 4, img2 += stride_img2 * 4) { 92 for (j = 0; j <= width - 8; j += 4) { 93 double v = ssim_8x8(img1 + j, stride_img1, img2 + j, stride_img2); 94 ssim_total += v; 95 samples++; 96 } 97 } 98 ssim_total /= samples; 99 return ssim_total; 100 } 101 102 #if CONFIG_INTERNAL_STATS 103 void aom_lowbd_calc_ssim(const YV12_BUFFER_CONFIG *source, 104 const YV12_BUFFER_CONFIG *dest, double *weight, 105 double *fast_ssim) { 106 double abc[3]; 107 for (int i = 0; i < 3; ++i) { 108 const int is_uv = i > 0; 109 abc[i] = aom_ssim2(source->buffers[i], dest->buffers[i], 110 source->strides[is_uv], dest->strides[is_uv], 111 source->crop_widths[is_uv], source->crop_heights[is_uv]); 112 } 113 114 *weight = 1; 115 *fast_ssim = abc[0] * .8 + .1 * (abc[1] + abc[2]); 116 } 117 118 // traditional ssim as per: http://en.wikipedia.org/wiki/Structural_similarity 119 // 120 // Re working out the math -> 121 // 122 // ssim(x,y) = (2*mean(x)*mean(y) + c1)*(2*cov(x,y)+c2) / 123 // ((mean(x)^2+mean(y)^2+c1)*(var(x)+var(y)+c2)) 124 // 125 // mean(x) = sum(x) / n 126 // 127 // cov(x,y) = (n*sum(xi*yi)-sum(x)*sum(y))/(n*n) 128 // 129 // var(x) = (n*sum(xi*xi)-sum(xi)*sum(xi))/(n*n) 130 // 131 // ssim(x,y) = 132 // (2*sum(x)*sum(y)/(n*n) + c1)*(2*(n*sum(xi*yi)-sum(x)*sum(y))/(n*n)+c2) / 133 // (((sum(x)*sum(x)+sum(y)*sum(y))/(n*n) +c1) * 134 // ((n*sum(xi*xi) - sum(xi)*sum(xi))/(n*n)+ 135 // (n*sum(yi*yi) - sum(yi)*sum(yi))/(n*n)+c2))) 136 // 137 // factoring out n*n 138 // 139 // ssim(x,y) = 140 // (2*sum(x)*sum(y) + n*n*c1)*(2*(n*sum(xi*yi)-sum(x)*sum(y))+n*n*c2) / 141 // (((sum(x)*sum(x)+sum(y)*sum(y)) + n*n*c1) * 142 // (n*sum(xi*xi)-sum(xi)*sum(xi)+n*sum(yi*yi)-sum(yi)*sum(yi)+n*n*c2)) 143 // 144 // Replace c1 with n*n * c1 for the final step that leads to this code: 145 // The final step scales by 12 bits so we don't lose precision in the constants. 146 147 static double ssimv_similarity(const Ssimv *sv, int64_t n) { 148 // Scale the constants by number of pixels. 149 const int64_t c1 = (cc1 * n * n) >> 12; 150 const int64_t c2 = (cc2 * n * n) >> 12; 151 152 const double l = 1.0 * (2 * sv->sum_s * sv->sum_r + c1) / 153 (sv->sum_s * sv->sum_s + sv->sum_r * sv->sum_r + c1); 154 155 // Since these variables are unsigned sums, convert to double so 156 // math is done in double arithmetic. 157 const double v = (2.0 * n * sv->sum_sxr - 2 * sv->sum_s * sv->sum_r + c2) / 158 (n * sv->sum_sq_s - sv->sum_s * sv->sum_s + 159 n * sv->sum_sq_r - sv->sum_r * sv->sum_r + c2); 160 161 return l * v; 162 } 163 164 // The first term of the ssim metric is a luminance factor. 165 // 166 // (2*mean(x)*mean(y) + c1)/ (mean(x)^2+mean(y)^2+c1) 167 // 168 // This luminance factor is super sensitive to the dark side of luminance 169 // values and completely insensitive on the white side. check out 2 sets 170 // (1,3) and (250,252) the term gives ( 2*1*3/(1+9) = .60 171 // 2*250*252/ (250^2+252^2) => .99999997 172 // 173 // As a result in this tweaked version of the calculation in which the 174 // luminance is taken as percentage off from peak possible. 175 // 176 // 255 * 255 - (sum_s - sum_r) / count * (sum_s - sum_r) / count 177 // 178 static double ssimv_similarity2(const Ssimv *sv, int64_t n) { 179 // Scale the constants by number of pixels. 180 const int64_t c1 = (cc1 * n * n) >> 12; 181 const int64_t c2 = (cc2 * n * n) >> 12; 182 183 const double mean_diff = (1.0 * sv->sum_s - sv->sum_r) / n; 184 const double l = (255 * 255 - mean_diff * mean_diff + c1) / (255 * 255 + c1); 185 186 // Since these variables are unsigned, sums convert to double so 187 // math is done in double arithmetic. 188 const double v = (2.0 * n * sv->sum_sxr - 2 * sv->sum_s * sv->sum_r + c2) / 189 (n * sv->sum_sq_s - sv->sum_s * sv->sum_s + 190 n * sv->sum_sq_r - sv->sum_r * sv->sum_r + c2); 191 192 return l * v; 193 } 194 static void ssimv_parms(uint8_t *img1, int img1_pitch, uint8_t *img2, 195 int img2_pitch, Ssimv *sv) { 196 aom_ssim_parms_8x8(img1, img1_pitch, img2, img2_pitch, &sv->sum_s, &sv->sum_r, 197 &sv->sum_sq_s, &sv->sum_sq_r, &sv->sum_sxr); 198 } 199 200 double aom_get_ssim_metrics(uint8_t *img1, int img1_pitch, uint8_t *img2, 201 int img2_pitch, int width, int height, Ssimv *sv2, 202 Metrics *m, int do_inconsistency) { 203 double dssim_total = 0; 204 double ssim_total = 0; 205 double ssim2_total = 0; 206 double inconsistency_total = 0; 207 int i, j; 208 int c = 0; 209 double norm; 210 double old_ssim_total = 0; 211 // We can sample points as frequently as we like start with 1 per 4x4. 212 for (i = 0; i < height; 213 i += 4, img1 += img1_pitch * 4, img2 += img2_pitch * 4) { 214 for (j = 0; j < width; j += 4, ++c) { 215 Ssimv sv = { 0, 0, 0, 0, 0, 0 }; 216 double ssim; 217 double ssim2; 218 double dssim; 219 uint32_t var_new; 220 uint32_t var_old; 221 uint32_t mean_new; 222 uint32_t mean_old; 223 double ssim_new; 224 double ssim_old; 225 226 // Not sure there's a great way to handle the edge pixels 227 // in ssim when using a window. Seems biased against edge pixels 228 // however you handle this. This uses only samples that are 229 // fully in the frame. 230 if (j + 8 <= width && i + 8 <= height) { 231 ssimv_parms(img1 + j, img1_pitch, img2 + j, img2_pitch, &sv); 232 } 233 234 ssim = ssimv_similarity(&sv, 64); 235 ssim2 = ssimv_similarity2(&sv, 64); 236 237 sv.ssim = ssim2; 238 239 // dssim is calculated to use as an actual error metric and 240 // is scaled up to the same range as sum square error. 241 // Since we are subsampling every 16th point maybe this should be 242 // *16 ? 243 dssim = 255 * 255 * (1 - ssim2) / 2; 244 245 // Here I introduce a new error metric: consistency-weighted 246 // SSIM-inconsistency. This metric isolates frames where the 247 // SSIM 'suddenly' changes, e.g. if one frame in every 8 is much 248 // sharper or blurrier than the others. Higher values indicate a 249 // temporally inconsistent SSIM. There are two ideas at work: 250 // 251 // 1) 'SSIM-inconsistency': the total inconsistency value 252 // reflects how much SSIM values are changing between this 253 // source / reference frame pair and the previous pair. 254 // 255 // 2) 'consistency-weighted': weights de-emphasize areas in the 256 // frame where the scene content has changed. Changes in scene 257 // content are detected via changes in local variance and local 258 // mean. 259 // 260 // Thus the overall measure reflects how inconsistent the SSIM 261 // values are, over consistent regions of the frame. 262 // 263 // The metric has three terms: 264 // 265 // term 1 -> uses change in scene Variance to weight error score 266 // 2 * var(Fi)*var(Fi-1) / (var(Fi)^2+var(Fi-1)^2) 267 // larger changes from one frame to the next mean we care 268 // less about consistency. 269 // 270 // term 2 -> uses change in local scene luminance to weight error 271 // 2 * avg(Fi)*avg(Fi-1) / (avg(Fi)^2+avg(Fi-1)^2) 272 // larger changes from one frame to the next mean we care 273 // less about consistency. 274 // 275 // term3 -> measures inconsistency in ssim scores between frames 276 // 1 - ( 2 * ssim(Fi)*ssim(Fi-1)/(ssim(Fi)^2+sssim(Fi-1)^2). 277 // 278 // This term compares the ssim score for the same location in 2 279 // subsequent frames. 280 var_new = sv.sum_sq_s - sv.sum_s * sv.sum_s / 64; 281 var_old = sv2[c].sum_sq_s - sv2[c].sum_s * sv2[c].sum_s / 64; 282 mean_new = sv.sum_s; 283 mean_old = sv2[c].sum_s; 284 ssim_new = sv.ssim; 285 ssim_old = sv2[c].ssim; 286 287 if (do_inconsistency) { 288 // We do the metric once for every 4x4 block in the image. Since 289 // we are scaling the error to SSE for use in a psnr calculation 290 // 1.0 = 4x4x255x255 the worst error we can possibly have. 291 static const double kScaling = 4. * 4 * 255 * 255; 292 293 // The constants have to be non 0 to avoid potential divide by 0 294 // issues other than that they affect kind of a weighting between 295 // the terms. No testing of what the right terms should be has been 296 // done. 297 static const double c1 = 1, c2 = 1, c3 = 1; 298 299 // This measures how much consistent variance is in two consecutive 300 // source frames. 1.0 means they have exactly the same variance. 301 const double variance_term = 302 (2.0 * var_old * var_new + c1) / 303 (1.0 * var_old * var_old + 1.0 * var_new * var_new + c1); 304 305 // This measures how consistent the local mean are between two 306 // consecutive frames. 1.0 means they have exactly the same mean. 307 const double mean_term = 308 (2.0 * mean_old * mean_new + c2) / 309 (1.0 * mean_old * mean_old + 1.0 * mean_new * mean_new + c2); 310 311 // This measures how consistent the ssims of two 312 // consecutive frames is. 1.0 means they are exactly the same. 313 double ssim_term = 314 pow((2.0 * ssim_old * ssim_new + c3) / 315 (ssim_old * ssim_old + ssim_new * ssim_new + c3), 316 5); 317 318 double this_inconsistency; 319 320 // Floating point math sometimes makes this > 1 by a tiny bit. 321 // We want the metric to scale between 0 and 1.0 so we can convert 322 // it to an snr scaled value. 323 if (ssim_term > 1) ssim_term = 1; 324 325 // This converts the consistency metric to an inconsistency metric 326 // ( so we can scale it like psnr to something like sum square error. 327 // The reason for the variance and mean terms is the assumption that 328 // if there are big changes in the source we shouldn't penalize 329 // inconsistency in ssim scores a bit less as it will be less visible 330 // to the user. 331 this_inconsistency = (1 - ssim_term) * variance_term * mean_term; 332 333 this_inconsistency *= kScaling; 334 inconsistency_total += this_inconsistency; 335 } 336 sv2[c] = sv; 337 ssim_total += ssim; 338 ssim2_total += ssim2; 339 dssim_total += dssim; 340 341 old_ssim_total += ssim_old; 342 } 343 old_ssim_total += 0; 344 } 345 346 norm = 1. / (width / 4) / (height / 4); 347 ssim_total *= norm; 348 ssim2_total *= norm; 349 m->ssim2 = ssim2_total; 350 m->ssim = ssim_total; 351 if (old_ssim_total == 0) inconsistency_total = 0; 352 353 m->ssimc = inconsistency_total; 354 355 m->dssim = dssim_total; 356 return inconsistency_total; 357 } 358 #endif // CONFIG_INTERNAL_STATS 359 360 #if CONFIG_AV1_HIGHBITDEPTH 361 void aom_highbd_ssim_parms_8x8_c(const uint16_t *s, int sp, const uint16_t *r, 362 int rp, uint32_t *sum_s, uint32_t *sum_r, 363 uint32_t *sum_sq_s, uint32_t *sum_sq_r, 364 uint32_t *sum_sxr) { 365 int i, j; 366 for (i = 0; i < 8; i++, s += sp, r += rp) { 367 for (j = 0; j < 8; j++) { 368 *sum_s += s[j]; 369 *sum_r += r[j]; 370 *sum_sq_s += s[j] * s[j]; 371 *sum_sq_r += r[j] * r[j]; 372 *sum_sxr += s[j] * r[j]; 373 } 374 } 375 } 376 377 static double highbd_ssim_8x8(const uint16_t *s, int sp, const uint16_t *r, 378 int rp, uint32_t bd, uint32_t shift) { 379 uint32_t sum_s = 0, sum_r = 0, sum_sq_s = 0, sum_sq_r = 0, sum_sxr = 0; 380 aom_highbd_ssim_parms_8x8(s, sp, r, rp, &sum_s, &sum_r, &sum_sq_s, &sum_sq_r, 381 &sum_sxr); 382 return similarity(sum_s >> shift, sum_r >> shift, sum_sq_s >> (2 * shift), 383 sum_sq_r >> (2 * shift), sum_sxr >> (2 * shift), 64, bd); 384 } 385 386 double aom_highbd_ssim2(const uint8_t *img1, const uint8_t *img2, 387 int stride_img1, int stride_img2, int width, int height, 388 uint32_t bd, uint32_t shift) { 389 int i, j; 390 int samples = 0; 391 double ssim_total = 0; 392 393 // sample point start with each 4x4 location 394 for (i = 0; i <= height - 8; 395 i += 4, img1 += stride_img1 * 4, img2 += stride_img2 * 4) { 396 for (j = 0; j <= width - 8; j += 4) { 397 double v = highbd_ssim_8x8(CONVERT_TO_SHORTPTR(img1 + j), stride_img1, 398 CONVERT_TO_SHORTPTR(img2 + j), stride_img2, bd, 399 shift); 400 ssim_total += v; 401 samples++; 402 } 403 } 404 ssim_total /= samples; 405 return ssim_total; 406 } 407 408 #if CONFIG_INTERNAL_STATS 409 void aom_highbd_calc_ssim(const YV12_BUFFER_CONFIG *source, 410 const YV12_BUFFER_CONFIG *dest, double *weight, 411 uint32_t bd, uint32_t in_bd, double *fast_ssim) { 412 assert(bd >= in_bd); 413 uint32_t shift = bd - in_bd; 414 415 double abc[3]; 416 for (int i = 0; i < 3; ++i) { 417 const int is_uv = i > 0; 418 abc[i] = aom_highbd_ssim2(source->buffers[i], dest->buffers[i], 419 source->strides[is_uv], dest->strides[is_uv], 420 source->crop_widths[is_uv], 421 source->crop_heights[is_uv], in_bd, shift); 422 } 423 424 weight[0] = 1; 425 fast_ssim[0] = abc[0] * .8 + .1 * (abc[1] + abc[2]); 426 427 if (bd > in_bd) { 428 // Compute SSIM based on stream bit depth 429 shift = 0; 430 for (int i = 0; i < 3; ++i) { 431 const int is_uv = i > 0; 432 abc[i] = aom_highbd_ssim2(source->buffers[i], dest->buffers[i], 433 source->strides[is_uv], dest->strides[is_uv], 434 source->crop_widths[is_uv], 435 source->crop_heights[is_uv], bd, shift); 436 } 437 438 weight[1] = 1; 439 fast_ssim[1] = abc[0] * .8 + .1 * (abc[1] + abc[2]); 440 } 441 } 442 #endif // CONFIG_INTERNAL_STATS 443 #endif // CONFIG_AV1_HIGHBITDEPTH 444 445 #if CONFIG_INTERNAL_STATS 446 void aom_calc_ssim(const YV12_BUFFER_CONFIG *orig, 447 const YV12_BUFFER_CONFIG *recon, const uint32_t bit_depth, 448 const uint32_t in_bit_depth, int is_hbd, double *weight, 449 double *frame_ssim2) { 450 #if CONFIG_AV1_HIGHBITDEPTH 451 if (is_hbd) { 452 aom_highbd_calc_ssim(orig, recon, weight, bit_depth, in_bit_depth, 453 frame_ssim2); 454 return; 455 } 456 #else 457 (void)bit_depth; 458 (void)in_bit_depth; 459 (void)is_hbd; 460 #endif // CONFIG_AV1_HIGHBITDEPTH 461 aom_lowbd_calc_ssim(orig, recon, weight, frame_ssim2); 462 } 463 #endif // CONFIG_INTERNAL_STATS