discrete spectrum is computed on unity cube, no spatial scaling required velocity field is computed for u_rms = 1, hence uPhysField subsequently has to be scaled by the magnitude of the fluctuations, e.g. uPhysField *= m_UInfinity
1052 {
1053 DEBUG("computeEnergySpectrum entry", MAIA_DEBUG_TRACE_IN);
1054 TRACE();
1055
1056 const MInt size = nx * ny * nz;
1058 const ptrdiff_t rank = 3;
1059 if(nx % 2 != 0 || ny % 2 != 0 || nz % 2 != 0) {
1060 std::stringstream errorMessage;
1061 errorMessage << " FFTInit: domainsize must NOT be an odd number! " << nx << " " << ny << " " << nz << std::endl;
1062 mTerm(1, AT_, errorMessage.str());
1063 }
1064 if(howmany < 3)
mTerm(1, AT_,
"expecting at least 3D velocity field");
1065
1066 m_log <<
" --- initializing FFTW --- " << std::endl;
1067 m_log <<
" domain size = " << nx <<
"x" << ny <<
"x" << nz <<
" (" << howmany <<
")" << std::endl;
1068
1069
1072 MPI_Comm_rank(comm, &domainId);
1073 MPI_Comm_size(comm, &noDomains);
1074 MPI_Comm MPI_COMM_FFTW;
1076 maxRank =
mMin(nz, noDomains);
1077 while(nz % maxRank != 0) {
1078 maxRank--;
1079 }
1080 MInt color = (domainId < maxRank) ? 0 : MPI_UNDEFINED;
1081 MPI_Comm_split(comm, color, domainId, &MPI_COMM_FFTW, AT_,
"MPI_COMM_FFTW");
1082 m_log <<
"no ranks for fft: " << maxRank << std::endl;
1083
1085 for(
MInt i = 0; i < locSize; i++) {
1086 urms0 +=
POW2(velocity(i, 0)) +
POW2(velocity(i, 1)) +
POW2(velocity(i, 2));
1087 }
1088 MPI_Allreduce(MPI_IN_PLACE, &urms0, 1, MPI_DOUBLE, MPI_SUM, comm, AT_,
"MPI_IN_PLACE",
"urms0");
1089 urms0 = sqrt(F1B3 * urms0 / fsize);
1090
1091 MInt fftLocSize = 0;
1092 ptrdiff_t alloc_local = 0, local_n0 = 0, local_0_start = 0;
1093 fftw_complex* hatField = nullptr;
1094 fftw_plan plan;
1095
1096 const MFloat time0 = MPI_Wtime();
1097
1098 const ptrdiff_t n[3] = {nx, ny, nz};
1099#ifndef MAIA_WINDOWS
1100 alloc_local = (domainId < maxRank) ? fftw_mpi_local_size_many(rank, n, howmany, FFTW_MPI_DEFAULT_BLOCK, MPI_COMM_FFTW,
1101 &local_n0, &local_0_start)
1102 : 1;
1103#endif
1104
1106 if(domainId < maxRank) {
1107 hatField = &hatFieldMem[0];
1108 } else {
1109 alloc_local = 0;
1110 local_n0 = 0;
1111 local_0_start = nx;
1112 }
1113
1117 noRecv.fill(0);
1118 noSend.fill(0);
1119 MLong locOffset = (domainId < maxRank) ? ((
MInt)local_0_start) * ny * nz : size;
1120
1121 MPI_Allgather(&locOffset, 1, MPI_LONG, &offsets[0], 1, MPI_LONG, comm, AT_,
"locOffset",
"offsets[0]");
1122 offsets(noDomains) = size;
1123 if(domainId < maxRank) {
1124 if(offsets(domainId) != locOffset || offsets(domainId + 1) != ((
MInt)(local_0_start + local_n0)) * ny * nz)
1125 mTerm(1, AT_,
"wrong size 0");
1126 }
1127 for(
MInt i = 0; i < locSize; i++) {
1129 MInt nghbrDomain =
mMin(maxRank - 1, j / (size / maxRank));
1130 while(j < offsets(nghbrDomain) || j >= offsets(nghbrDomain + 1)) {
1131 if(j < offsets(nghbrDomain)) nghbrDomain--;
1132 if(j >= offsets(nghbrDomain + 1)) nghbrDomain++;
1133 }
1134 if(nghbrDomain >= maxRank)
mTerm(1, AT_,
"wrong domain");
1135 noSend(nghbrDomain)++;
1136 }
1137 MPI_Alltoall(&noSend[0], 1, MPI_INT, &noRecv[0], 1, MPI_INT, comm, AT_,
"noSend[0]",
"noRecv[0]");
1142 for(
MInt i = 0; i < noDomains; i++) {
1143 recvOffsets(i) = recvSize;
1144 sendOffsets(i) = sendSize;
1145 sendSize += noSend(i);
1146 recvSize += noRecv(i);
1147 }
1148 recvOffsets(noDomains) = recvSize;
1149 sendOffsets(noDomains) = sendSize;
1154 noSend.fill(0);
1155 for(
MInt i = 0; i < locSize; i++) {
1157 MInt nghbrDomain =
mMin(maxRank - 1, j / (size / maxRank));
1158 while(j < offsets(nghbrDomain) || j >= offsets(nghbrDomain + 1)) {
1159 if(j < offsets(nghbrDomain)) nghbrDomain--;
1160 if(j >= offsets(nghbrDomain + 1)) nghbrDomain++;
1161 }
1162 if(nghbrDomain >= maxRank)
mTerm(1, AT_,
"wrong domain");
1163 if(sendOffsets(nghbrDomain) + noSend(nghbrDomain) >= sendOffsets(nghbrDomain + 1))
mTerm(1, AT_,
"wrong size");
1164 for(
MInt k = 0; k < howmany; k++) {
1165 sendData(sendOffsets(nghbrDomain) + noSend(nghbrDomain), k) = velocity(i, k);
1166 }
1167 sendIndices(sendOffsets(nghbrDomain) + noSend(nghbrDomain)) = j;
1168 if(j < offsets(nghbrDomain) || j >= offsets(nghbrDomain + 1))
mTerm(1, AT_,
"wrong size 01");
1169 noSend(nghbrDomain)++;
1170 }
1171 for(
MInt i = 0; i < noDomains; i++) {
1172 if(noSend(i) != sendOffsets(i + 1) - sendOffsets(i))
mTerm(1, AT_,
"wrong size 1");
1173 if(noRecv(i) != recvOffsets(i + 1) - recvOffsets(i))
mTerm(1, AT_,
"wrong size 1");
1174 }
1175
1177 sendReq.fill(MPI_REQUEST_NULL);
1179 for(
MInt i = 0; i < noDomains; i++) {
1180 if(noSend(i) == 0) continue;
1181 MPI_Issend(&sendData[howmany * sendOffsets(i)], howmany * noSend(i), MPI_DOUBLE, i, 123, comm, &sendReq[cnt], AT_,
1182 "sendData[howmany * sendOffsets(i)]");
1183 cnt++;
1184 }
1185 for(
MInt i = 0; i < noDomains; i++) {
1186 if(noRecv(i) == 0) continue;
1187 MPI_Recv(&recvData[howmany * recvOffsets(i)], howmany * noRecv(i), MPI_DOUBLE, i, 123, comm, MPI_STATUS_IGNORE, AT_,
1188 "recvData[howmany * recvOffsets(i)]");
1189 }
1190 if(cnt > 0)
MPI_Waitall(cnt, &sendReq[0], MPI_STATUSES_IGNORE, AT_);
1191 sendReq.fill(MPI_REQUEST_NULL);
1192 cnt = 0;
1193 for(
MInt i = 0; i < noDomains; i++) {
1194 if(noSend(i) == 0) continue;
1195 MPI_Issend(&sendIndices[sendOffsets(i)], noSend(i), MPI_INT, i, 124, comm, &sendReq[cnt], AT_,
1196 "sendIndices[sendOffsets(i)]");
1197 cnt++;
1198 }
1199 for(
MInt i = 0; i < noDomains; i++) {
1200 if(noRecv(i) == 0) continue;
1201 MPI_Recv(&recvIndices[recvOffsets(i)], noRecv(i), MPI_INT, i, 124, comm, MPI_STATUS_IGNORE, AT_,
1202 "recvIndices[recvOffsets(i)]");
1203 }
1204 if(cnt > 0)
MPI_Waitall(cnt, &sendReq[0], MPI_STATUSES_IGNORE, AT_);
1205
1206 if(domainId < maxRank) {
1207 for(
MInt i = 0; i < recvSize; i++) {
1208 MInt j = recvIndices(i);
1209 if(j < ((
MInt)local_0_start) * ny * nz || j >= ((
MInt)(local_0_start + local_n0)) * ny * nz)
1210 mTerm(1, AT_,
"wrong size 2");
1211 MInt pos = j - (((
MInt)local_0_start) * ny * nz);
1212 for(
MInt k = 0; k < howmany; k++) {
1213 hatField[howmany * pos + k][0] = recvData(i, k);
1214 hatField[howmany * pos + k][1] = F0;
1215 }
1216 }
1217 fftLocSize = recvSize;
1218
1220 for(
MInt i = 0; i < fftLocSize; i++) {
1221 for(
MInt k = 0; k < 3; k++) {
1222 couplcheck -= hatField[howmany * i + k][0] * hatField[howmany * i + 3 + k][0];
1223 }
1224 }
1225 MPI_Allreduce(MPI_IN_PLACE, &couplcheck, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_FFTW, AT_,
"MPI_IN_PLACE",
"couplcheck");
1226 if(domainId == 0)
1227 std::cerr <<
"couplecheck fft " << couplcheck <<
" " << couplcheck *
POW3(0.25 / 64.0) << std::endl;
1228 }
1229
1230
1231 if(domainId < maxRank) {
1232#ifndef MAIA_WINDOWS
1233 plan = fftw_mpi_plan_many_dft(rank, n, howmany, FFTW_MPI_DEFAULT_BLOCK, FFTW_MPI_DEFAULT_BLOCK, hatField, hatField,
1234 MPI_COMM_FFTW, FFTW_FORWARD, FFTW_ESTIMATE);
1235 fftw_execute(plan);
1236 fftw_destroy_plan(plan);
1237#endif
1238 for(
MInt i = 0; i < fftLocSize; i++) {
1239 for(
MInt j = 0; j < howmany; j++) {
1240 hatField[howmany * i + j][0] /= fsize;
1241 hatField[howmany * i + j][1] /= fsize;
1242 }
1243 }
1244 }
1245
1246 const MFloat time1 = MPI_Wtime();
1247
1248 m_log <<
"parallel fftw time " << time1 - time0 << std::endl;
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258 static const MInt referenceCubeSize =
1259 ((
Context::propertyExists(
"referenceCubeSize")) ? Context::getBasicProperty<MInt>(
"referenceCubeSize", AT_) : 1);
1260
1262
1264
1265 const MFloat k0 = F2 * PI / referenceCubeSize;
1270 if(kMinSpec > 0 && kMaxSpec > 0) {
1271 noBins = kMaxSpec;
1272 kMin = static_cast<float>(kMinSpec) * k0;
1273 kMax = static_cast<float>(kMaxSpec) * k0;
1274 deltaK = (kMax - kMin) / ((
MFloat)(kMaxSpec - 1));
1275 } else {
1276 noBins =
mMax(nx,
mMax(ny, nz)) / 2;
1277 kMin = F1 * k0;
1278 kMax = noBins * k0;
1279 deltaK = (kMax - kMin) / ((
MFloat)(noBins - 1));
1280 }
1281
1282
1283
1294 spectrumBnd[0] = kMin - F1B2 * deltaK;
1295 const MFloat spectrumBnd00 = kMin - F3B4 * deltaK;
1296 for(
MInt i = 0; i < noBins; i++) {
1297 spectrumBnd[i + 1] = spectrumBnd[i] + deltaK;
1298 }
1299 spectrum.fill(F0);
1300 coupling.fill(F0);
1301 transfer.fill(F0);
1302 transfer2.fill(F0);
1303 rate.fill(F0);
1304 flux.fill(F0);
1305 flux2.fill(F0);
1306 spectrumSum.fill(F0);
1307 spectrumSum2.fill(F0);
1308
1309
1310 static const MBool computeTransferRate =
1311 ((
Context::propertyExists(
"computeTransferRate")) ? Context::getBasicProperty<MBool>(
"computeTransferRate", AT_)
1312 : false);
1313 if(computeTransferRate) {
1316 MPI_Allgather(&fftLocSize, 1, MPI_INT, &noDatPerDomain[0], 1, MPI_INT, comm, AT_,
"fftLocSize",
1317 "noDatPerDomain[0]");
1318 dataOffsets.fill(0);
1319 for(
MInt d = 0; d < noDomains; d++) {
1320 dataOffsets(d + 1) = dataOffsets(d) + noDatPerDomain(d);
1321 }
1322
1323 MPI_Request sendReqLoc = MPI_REQUEST_NULL;
1324 fftw_complex* uHatGlobal = fftw_alloc_complex(3 * size);
1325 fftw_complex* uHatGlobalTmp = nullptr;
1326 for(
MInt i = 0; i < fftLocSize; i++) {
1327 for(
MInt j = 0; j < 3; j++) {
1328 uHatGlobal[3 * i + j][0] = hatField[howmany * i + j][0];
1329 uHatGlobal[3 * i + j][1] = hatField[howmany * i + j][1];
1330 }
1331 }
1332 if(fftLocSize > 0) {
1333 MPI_Issend(&uHatGlobal[0], 6 * fftLocSize, MPI_DOUBLE, 0, 18, comm, &sendReqLoc, AT_,
"uHatGlobal[0]");
1334 }
1335 if(domainId == 0) {
1336 uHatGlobalTmp = fftw_alloc_complex(3 * size);
1337 for(
MInt d = 0; d < noDomains; d++) {
1338 if(noDatPerDomain[d] > 0) {
1339 MPI_Recv(&(uHatGlobalTmp[3 * dataOffsets[d]][0]), 6 * noDatPerDomain[d], MPI_DOUBLE, d, 18, comm,
1340 MPI_STATUSES_IGNORE, AT_, "(uHatGlobalTmp[3 * dataOffsets[d]][0])");
1341 }
1342 }
1343 }
1344 if(fftLocSize > 0) {
1345 MPI_Wait(&sendReqLoc, MPI_STATUSES_IGNORE, AT_);
1346 }
1347
1348 if(domainId == 0) {
1349
1350 for(
MInt i0 = 0; i0 < nx; i0++) {
1351 for(
MInt i1 = 0; i1 < ny; i1++) {
1352 for(
MInt i2 = 0; i2 < nz; i2++) {
1353 MInt j0 = (i0 > nx / 2) ? i0 - nx : i0;
1354 MInt j1 = (i1 > ny / 2) ? i1 - ny : i1;
1355 MInt j2 = (i2 > nz / 2) ? i2 - nz : i2;
1356 j0 += nx / 2 - 1;
1357 j1 += nx / 2 - 1;
1358 j2 += nx / 2 - 1;
1361 for(
MInt j = 0; j < 3; j++) {
1362 uHatGlobal[3 * pos1 + j][0] = uHatGlobalTmp[3 * pos0 + j][0];
1363 uHatGlobal[3 * pos1 + j][1] = uHatGlobalTmp[3 * pos0 + j][1];
1364 }
1365 }
1366 }
1367 }
1368 fftw_free(uHatGlobalTmp);
1369 }
1370
1371 MPI_Bcast(&uHatGlobal[0], 6 * size, MPI_DOUBLE, 0, comm, AT_,
"uHatGlobal");
1372
1373
1374
1375
1376
1377
1378
1379 MInt noWaveNumbers = 0;
1380 MLong weightSum = 0;
1381 for(
MInt pos = 0; pos < size; pos++) {
1383 MInt i1 = (pos / nz) % ny;
1384 MInt i0 = pos / (ny * nz);
1385 if(pos != i2 + nz * (i1 + ny * (i0)))
mTerm(1, AT_,
"index mismatch");
1387 MInt j0 = i0 - nx / 2 + 1;
1388 MInt j1 = i1 - ny / 2 + 1;
1389 MInt j2 = i2 - nz / 2 + 1;
1390 k[0] = ((
MFloat)j0) * k0;
1391 k[1] = ((
MFloat)j1) * k0;
1392 k[2] = ((
MFloat)j2) * k0;
1393
1394 MFloat kAbs = sqrt(k[0] * k[0] + k[1] * k[1] + k[2] * k[2]);
1395 if(kAbs > spectrumBnd[noBins] + 1e-12) continue;
1396 if(kAbs < spectrumBnd[0] - 1e-12) continue;
1397
1398 MInt p0Min = std::max(0, i0 - nx / 2);
1399 MInt p0Max = std::min(nx, i0 + nx / 2);
1400 MInt p1Min = std::max(0, i1 - ny / 2);
1401 MInt p1Max = std::min(ny, i1 + ny / 2);
1402 MInt p2Min = std::max(0, i2 - nz / 2);
1403 MInt p2Max = std::min(nz, i2 + nz / 2);
1404 MLong weight = ((p0Max - p0Min) * (p1Max - p1Min) * (p2Max - p2Min));
1405
1406 weightSum += weight;
1407 noWaveNumbers++;
1408 }
1409
1412 MLong domainSize = weightSum / noDomainsLong;
1415 noWaveNumbers = 0;
1416 weightSum = 0;
1417 for(
MInt pos = 0; pos < size; pos++) {
1419 MInt i1 = (pos / nz) % ny;
1420 MInt i0 = pos / (ny * nz);
1421 if(pos != i2 + nz * (i1 + ny * (i0)))
mTerm(1, AT_,
"index mismatch");
1423 MInt j0 = i0 - nx / 2 + 1;
1424 MInt j1 = i1 - ny / 2 + 1;
1425 MInt j2 = i2 - nz / 2 + 1;
1426 k[0] = ((
MFloat)j0) * k0;
1427 k[1] = ((
MFloat)j1) * k0;
1428 k[2] = ((
MFloat)j2) * k0;
1429
1430 MFloat kAbs = sqrt(k[0] * k[0] + k[1] * k[1] + k[2] * k[2]);
1431 if(kAbs > spectrumBnd[noBins] + 1e-12) continue;
1432 if(kAbs < spectrumBnd[0] - 1e-12) continue;
1433
1434 MInt p0Min = std::max(0, i0 - nx / 2);
1435 MInt p0Max = std::min(nx, i0 + nx / 2);
1436 MInt p1Min = std::max(0, i1 - ny / 2);
1437 MInt p1Max = std::min(ny, i1 + ny / 2);
1438 MInt p2Min = std::max(0, i2 - nz / 2);
1439 MInt p2Max = std::min(nz, i2 + nz / 2);
1440 MLong weight = ((p0Max - p0Min) * (p1Max - p1Min) * (p2Max - p2Min));
1441
1442 if(weightSum <= (domainSize * domainIdLong) && (weightSum + weight) > (domainSize * domainIdLong)) posMin = pos;
1443 if(weightSum <= (domainSize * (domainIdLong + 1)) && (weightSum + weight) > (domainSize * (domainIdLong + 1)))
1444 posMax = pos;
1445
1446 weightSum += weight;
1447 noWaveNumbers++;
1448 }
1449
1450 weightSum = 0;
1451 for(
MInt pos = posMin; pos < posMax; pos++) {
1452 const MInt i2 = pos % nz;
1453 const MInt i1 = (pos / nz) % ny;
1454 const MInt i0 = pos / (ny * nz);
1455
1456 const MInt j0 = i0 - nx / 2 + 1;
1457 const MInt j1 = i1 - ny / 2 + 1;
1458 const MInt j2 = i2 - nz / 2 + 1;
1460 const MFloat kAbs2 =
mMax(1e-12, k[0] * k[0] + k[1] * k[1] + k[2] * k[2]);
1461 const MFloat kAbs =
mMax(1e-12, sqrt(k[0] * k[0] + k[1] * k[1] + k[2] * k[2]));
1462
1463 if(kAbs > spectrumBnd[noBins] + 1e-12) continue;
1464 const MInt bin = ((
MInt)((kAbs - spectrumBnd[0] + 1e-12) / deltaK));
1465 const MInt bin2 = ((
MInt)((kAbs - spectrumBnd00 + 1e-12) / deltaK));
1466
1467 const MFloat kjl[3][3] = {
1468 {1.0 - (k[0] * k[0] / kAbs2), 0.0 - (k[0] * k[1] / kAbs2), 0.0 - (k[0] * k[2] / kAbs2)},
1469 {0.0 - (k[1] * k[0] / kAbs2), 1.0 - (k[1] * k[1] / kAbs2), 0.0 - (k[1] * k[2] / kAbs2)},
1470 {0.0 - (k[2] * k[0] / kAbs2), 0.0 - (k[2] * k[1] / kAbs2), 1.0 - (k[2] * k[2] / kAbs2)}};
1471
1472 const MInt p0Min = std::max(0, i0 - nx / 2);
1473 const MInt p0Max = std::min(nx, i0 + nx / 2);
1474 const MInt p1Min = std::max(0, i1 - ny / 2);
1475 const MInt p1Max = std::min(ny, i1 + ny / 2);
1476 const MInt p2Min = std::max(0, i2 - nz / 2);
1477 const MInt p2Max = std::min(nz, i2 + nz / 2);
1478
1480
1481 for(
MInt p0 = p0Min; p0 < p0Max; p0++) {
1482 for(
MInt p1 = p1Min; p1 < p1Max; p1++) {
1483 for(
MInt p2 = p2Min; p2 < p2Max; p2++) {
1484 const MInt pos1 = p2 + nz * (p1 + ny * (p0));
1485 const MInt s0 = i0 - p0 + nx / 2 - 1;
1486 const MInt s1 = i1 - p1 + ny / 2 - 1;
1487 const MInt s2 = i2 - p2 + nz / 2 - 1;
1488 const MInt pos2 = s2 + nz * (s1 + ny * (s0));
1489
1490 for(
MInt i = 0; i < 3; i++) {
1491 for(
MInt j = 0; j < 3; j++) {
1492 for(
MInt l = 0; l < 3; l++) {
1493 trans += k[i] * kjl[j][l]
1494 *
triadImag(uHatGlobal[3 * pos1 + l], uHatGlobal[3 * pos2 + i], uHatGlobal[3 * pos + j]);
1495 }
1496 }
1497 }
1498 }
1499 }
1500 }
1501
1502 if(bin > -1 && bin < noBins) transfer(bin) += trans;
1503 if(bin2 > -1 && bin2 < noBins) transfer2(bin2) += trans;
1504 }
1505
1506 MPI_Allreduce(MPI_IN_PLACE, &transfer[0], noBins, MPI_DOUBLE, MPI_SUM, comm, AT_,
"MPI_IN_PLACE",
"transfer[0]");
1507 MPI_Allreduce(MPI_IN_PLACE, &transfer2[0], noBins, MPI_DOUBLE, MPI_SUM, comm, AT_,
"MPI_IN_PLACE",
"transfer2[0]");
1508 fftw_free(uHatGlobal);
1509 }
1510
1511 const MFloat time2 = MPI_Wtime();
1512
1513 if(domainId < maxRank) {
1514 for(
MInt i0 = (
MInt)local_0_start; i0 < (
MInt)(local_0_start + local_n0); i0++) {
1515 for(
MInt i1 = 0; i1 < ny; i1++) {
1516 for(
MInt i2 = 0; i2 < nz; i2++) {
1518 MInt j0 = (i0 > nx / 2) ? i0 - nx : i0;
1519 MInt j1 = (i1 > ny / 2) ? i1 - ny : i1;
1520 MInt j2 = (i2 > nz / 2) ? i2 - nz : i2;
1521 k[0] = ((
MFloat)j0) * k0;
1522 k[1] = ((
MFloat)j1) * k0;
1523 k[2] = ((
MFloat)j2) * k0;
1524
1525 MInt pos = i2 + nz * (i1 + ny * (i0 - ((
MInt)local_0_start)));
1526 if(pos < 0 || pos >= fftLocSize)
mTerm(1, AT_,
"wrong size 3");
1527
1528 MFloat kAbs = sqrt(k[0] * k[0] + k[1] * k[1] + k[2] * k[2]);
1532 for(
MInt q = 0; q < 3; q++) {
1533 eng += F1B2 * (
POW2(hatField[howmany * pos + q][0]) +
POW2(hatField[howmany * pos + q][1]));
1534 }
1535
1536
1537
1538 if(howmany > 3) {
1539 for(
MInt q = 0; q < 3; q++) {
1540 coupl += (hatField[howmany * pos + q][0] * hatField[howmany * pos + 3 + q][0]
1541 + hatField[howmany * pos + q][1] * hatField[howmany * pos + 3 + q][1]);
1542 dedt += (hatField[howmany * pos + 6 + q][0] * hatField[howmany * pos + q][0]
1543 + hatField[howmany * pos + 6 + q][1] * hatField[howmany * pos + q][1]);
1544 }
1545 }
1546 if(kAbs > spectrumBnd[noBins] + 1e-12) continue;
1547
1548 MInt bin = ((
MInt)((kAbs - spectrumBnd[0] + 1e-12) / deltaK));
1549 MInt bin2 = ((
MInt)((kAbs - spectrumBnd00 + 1e-12) / deltaK));
1550 if(bin > -1 && bin < noBins) {
1551 spectrum(bin) += eng;
1552 coupling(bin) += coupl;
1553 rate(bin) += dedt;
1554 spectrumSum(bin) += F1;
1555 }
1556 if(bin2 > -1 && bin2 < noBins) {
1557 spectrumSum2(bin2) += F1;
1558 }
1559 }
1560 }
1561 }
1562
1563 MPI_Allreduce(MPI_IN_PLACE, &spectrum[0], noBins, MPI_DOUBLE, MPI_SUM, MPI_COMM_FFTW, AT_,
"MPI_IN_PLACE",
1564 "spectrum[0]");
1565 MPI_Allreduce(MPI_IN_PLACE, &coupling[0], noBins, MPI_DOUBLE, MPI_SUM, MPI_COMM_FFTW, AT_,
"MPI_IN_PLACE",
1566 "coupling[0]");
1567 MPI_Allreduce(MPI_IN_PLACE, &rate[0], noBins, MPI_DOUBLE, MPI_SUM, MPI_COMM_FFTW, AT_,
"MPI_IN_PLACE",
"rate[0]");
1568 MPI_Allreduce(MPI_IN_PLACE, &spectrumSum[0], noBins, MPI_DOUBLE, MPI_SUM, MPI_COMM_FFTW, AT_,
"MPI_IN_PLACE",
1569 "spectrumSum[0]");
1570 MPI_Allreduce(MPI_IN_PLACE, &spectrumSum2[0], noBins, MPI_DOUBLE, MPI_SUM, MPI_COMM_FFTW, AT_,
"MPI_IN_PLACE",
1571 "spectrumSum2[0]");
1572
1573 for(
MInt i = 0; i < noBins; i++) {
1574 MFloat db = spectrumBnd00 - spectrumBnd[0];
1576 F4B3 * PI * (
POW3(spectrumBnd[i + 1]) -
POW3(spectrumBnd[i])) / (spectrumBnd[i + 1] - spectrumBnd[i]);
1577 MFloat vol2 = F4B3 * PI * (
POW3(spectrumBnd[i + 1] + db) -
POW3(spectrumBnd[i] + db))
1578 / (spectrumBnd[i + 1] - spectrumBnd[i]);
1579 spectrum(i) *= vol / (spectrumSum(i) *
POW3(k0));
1580 coupling(i) *= vol / (spectrumSum(i) *
POW3(k0));
1581 transfer(i) *= vol / (spectrumSum(i) *
POW3(k0));
1582 transfer2(i) *= vol2 / (spectrumSum2(i) *
POW3(k0));
1583 rate(i) *= vol / (spectrumSum(i) *
POW3(k0));
1584 }
1585
1586 for(
MInt i = 0; i < noBins; i++) {
1587 flux(i, 0) = F0;
1588 flux(i, 1) = F0;
1589 for(
MInt j = 0; j < i; j++) {
1590 flux(i, 0) += transfer(j) * (spectrumBnd[j + 1] - spectrumBnd[j]) / k0;
1591 }
1592 for(
MInt j = i; j < noBins; j++) {
1593 flux(i, 1) += transfer(j) * (spectrumBnd[j + 1] - spectrumBnd[j]) / k0;
1594 }
1595 flux2(i, 0) = F0;
1596 flux2(i, 1) = F0;
1597 for(
MInt j = 0; j < i; j++) {
1598 flux2(i, 0) += transfer2(j) * (spectrumBnd[j + 1] - spectrumBnd[j]) / k0;
1599 }
1600 for(
MInt j = i; j < noBins; j++) {
1601 flux2(i, 1) += transfer2(j) * (spectrumBnd[j + 1] - spectrumBnd[j]) / k0;
1602 }
1603 }
1604
1605 if(domainId == 0) {
1606 std::ofstream ofl;
1607 std::ofstream ofl2;
1608 ofl.open(
"./out/energySpectrum_00" + std::to_string(
globalTimeStep), std::ios_base::out | std::ios_base::trunc);
1609 ofl2.open(
"./out/energySpectrum_coarse_00" + std::to_string(
globalTimeStep),
1610 std::ios_base::out | std::ios_base::trunc);
1611 if(ofl.is_open() && ofl.good() && ofl2.is_open() && ofl2.good()) {
1620 for(
MInt i = 0; i < noBins; i++) {
1621 urms += spectrum(i) * (spectrumBnd[i + 1] - spectrumBnd[i]);
1622 }
1623 urms = sqrt(F2B3 * urms);
1624 for(
MInt i = 0; i < noBins; i++) {
1625 MFloat k = F1B2 * (spectrumBnd[i + 1] + spectrumBnd[i]);
1627 if(std::isnan(Ek) || std::isnan(coupling(i)) || std::isnan(rate(i))) continue;
1628 energy += (spectrumBnd[i + 1] - spectrumBnd[i]) * Ek;
1629 dissip += F2 * viscosity *
POW2(k) * (spectrumBnd[i + 1] - spectrumBnd[i]) * Ek;
1630 length += (F1B2 * PI /
POW2(urms)) * (spectrumBnd[i + 1] - spectrumBnd[i]) * Ek / (k);
1631 force += (spectrumBnd[i + 1] - spectrumBnd[i]) * coupling(i);
1632 dedt += (spectrumBnd[i + 1] - spectrumBnd[i]) * rate(i);
1633 transf += (spectrumBnd[i + 1] - spectrumBnd[i]) * transfer(i);
1634 transf2 += (spectrumBnd[i + 1] - spectrumBnd[i]) * transfer2(i);
1635 }
1636 ofl <<
"# Energy spectrum at time step " <<
globalTimeStep <<
", total energy Ek=" << energy
1637 << ", dissipation rate=" << dissip << ", u_rms=" << urms << " (" << urms0
1638 << "), integral length scale=" << length << ", Kolmogorov time scale=" << sqrt(viscosity / dissip)
1639 << ", coupling=" << force << ", dedt=" << dedt << ", transfer=" << transf << ", transfer2=" << transf2
1640 << std::endl;
1641 ofl << "# 1: wave number k" << std::endl;
1642 ofl << "# 2: energy E(k)" << std::endl;
1643 ofl << "# 3: viscous dissipation rate D(k)" << std::endl;
1644 ofl << "# 4: transfer rate T(k)" << std::endl;
1645 ofl << "# 5: coupling rate Psi(k)" << std::endl;
1646 ofl << "# 6: flux -F(k)" << std::endl;
1647 ofl << "# 7: flux F(k)" << std::endl;
1648 ofl << "# 8: dE/dt(k)" << std::endl;
1649 ofl << "# 9: alternative transfer rate T(k)" << std::endl;
1650 ofl << "# 10: alternative flux -F(k)" << std::endl;
1651 ofl << "# 11: alternative flux F(k)" << std::endl;
1652 ofl << "# 12: number of samples in corresponding bin" << std::endl;
1653 ofl << "# 13: lower boundary of corresponding bin" << std::endl;
1654 ofl << "# 14: upper boundary of corresponding bin" << std::endl;
1655 ofl << "# 15: number of samples other spectrum: " << std::endl;
1656 ofl << "# 16: 4 * PI * k^2" << std::endl;
1657 ofl << "# 17: 4/3 * PI * (upperBndry^3 - lowerBndry^3)" << std::endl;
1658 for(
MInt i = 0; i < noBins; i++) {
1659 MFloat k = F1B2 * (spectrumBnd[i + 1] + spectrumBnd[i]);
1661 MFloat coupl = coupling(i);
1662 MFloat trans = transfer(i);
1663 MFloat trans2 = transfer2(i);
1664 ofl << k / k0 <<
" " << Ek <<
" " << F2 *
POW2(k) * viscosity * Ek <<
" " << trans <<
" " << coupl <<
" "
1665 << flux(i, 0) << " " << flux(i, 1) << " " << rate(i) << " " << trans2 << " " << flux2(i, 0) << " "
1666 << flux2(i, 1) << " " << spectrumSum(i) << " " << spectrumBnd[i] << " " << spectrumBnd[i + 1] << " "
1667 << spectrumSum2(i) <<
" " << F4 * PI *
POW2(k) <<
" "
1668 << F4B3 * PI * (
POW3(spectrumBnd[i + 1]) -
POW3(spectrumBnd[i])) << std::endl;
1669 }
1670 for(
MInt i = 0; i < noBins / 2; i++) {
1671 const MInt i0 = 2 * i;
1672 const MInt i1 = 2 * i + 1;
1673 const MFloat k = F1B2 * (spectrumBnd[i1 + 1] + spectrumBnd[i0]);
1675 F4B3 * PI * (
POW3(spectrumBnd[i0 + 1]) -
POW3(spectrumBnd[i0])) / (spectrumBnd[i0 + 1] - spectrumBnd[i0]);
1677 F4B3 * PI * (
POW3(spectrumBnd[i1 + 1]) -
POW3(spectrumBnd[i1])) / (spectrumBnd[i1 + 1] - spectrumBnd[i1]);
1679 F4B3 * PI * (
POW3(spectrumBnd[i1 + 1]) -
POW3(spectrumBnd[i0])) / (spectrumBnd[i1 + 1] - spectrumBnd[i0]);
1680 MFloat f0 = spectrumSum(i0) * vol / (vol0 * (spectrumSum(i0) + spectrumSum(i1)));
1681 MFloat f1 = spectrumSum(i1) * vol / (vol1 * (spectrumSum(i0) + spectrumSum(i1)));
1682
1683 ofl2 << k / k0 << " " << f1 * spectrum(i1) + f0 * spectrum(i0) << " "
1684 << F2 *
POW2(k) * viscosity * (f1 * spectrum(i1) + f0 * spectrum(i0)) <<
" "
1685 << f1 * transfer(i1) + f0 * transfer(i0) << " " << f1 * coupling(i1) + f0 * coupling(i0) << " "
1686 << f1 * flux(i1, 0) + f0 * flux(i0, 0) << " " << f1 * flux(i1, 1) + f0 * flux(i0, 1) << " "
1687 << f1 * rate(i1) + f0 * rate(i0) << " " << f1 * transfer2(i1) + f0 * transfer2(i0) << " "
1688 << f1 * flux2(i1, 0) + f0 * flux2(i0, 0) << " " << f1 * flux2(i1, 1) + f0 * flux2(i0, 1) << " "
1689 << spectrumSum(i0) + spectrumSum(i1) << " " << spectrumBnd[i0] << " " << spectrumBnd[i1 + 1] << " "
1690 << spectrumSum2(i0) + spectrumSum2(i1) <<
" " << F4 * PI *
POW2(k) <<
" "
1691 << F4B3 * PI * (
POW3(spectrumBnd[i1 + 1]) -
POW3(spectrumBnd[i0])) << std::endl;
1692 }
1693 ofl.close();
1694 ofl2.close();
1695 }
1696 }
1697
1698
1699
1700 }
1701
1702 const MFloat time3 = MPI_Wtime();
1703
1704 if(domainId == 0)
1705 std::cerr << "fft time " << time1 - time0 << " " << time2 - time1 << " " << time3 - time2 << std::endl;
1706
1707 m_log <<
"fft finished" << std::endl;
1708
1709 DEBUG("computeEnergySpectrum return", MAIA_DEBUG_TRACE_OUT);
1710}
static MBool propertyExists(const MString &name, MInt solver=m_noSolvers)
This function checks if a property exists in general.
This class is a ScratchSpace.
constexpr Real POW3(const Real x)
constexpr Real POW2(const Real x)
constexpr T mMin(const T &x, const T &y)
constexpr T mMax(const T &x, const T &y)
int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *status, const MString &name, const MString &varname)
same as MPI_Recv
int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request, const MString &name, const MString &varname)
same as MPI_Issend
int MPI_Wait(MPI_Request *request, MPI_Status *status, const MString &name)
same as MPI_Wait
int MPI_Waitall(int count, MPI_Request *request, MPI_Status *status, const MString &name)
same as MPI_Waitall
int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, const MString &name, const MString &sndvarname, const MString &rcvvarname)
same as MPI_Allreduce
int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm, const MString &name, const MString &sndvarname, const MString &rcvvarname)
same as MPI_Alltoall
int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm, const MString &name, const MString &varname)
same as MPI_Comm_split, but updates the number of MPI communicators
int MPI_Bcast(void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm, const MString &name, const MString &varname)
same as MPI_Bcast
int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm, const MString &name, const MString &sndvarname, const MString &rcvvarname)
same as MPI_Allgather
void indices(const MInt i, const MInt n, MInt *const ids)
Calculate the 2D/3D indices for a given scalar id for accessing a field of n^nDim points.
MFloat triadImag(fftw_complex &a, fftw_complex &b, fftw_complex &c)
MInt getGlobalPosFFTW(MInt i0, MInt i1, MInt i2, MInt ny, MInt nz)