我正在尝试使用kinect跟踪3D打印对象并将一些纹理/ GUI投影到它们上 . 物体躺在 table 上，kinect和投影仪悬挂在天花板朝下 . 跟踪部件已经很好地工作并且给出了相对于kinect的对象的位置（x，y，z）和旋转（滚动，俯仰，偏转） .
现在我想在这些对象上投射一些纹理（或纯色） . 我的第一个想法是通过将对象网格放置在跟踪位置和旋转来重新构建Unity中的场景 . 现在我将统一摄像机放在投影机的位置，然后用投影机输出摄像机的图像 . 为了实现这个目的，我需要投影仪的外观（相对于kinect）和内在函数（f_x，f_y，c_x，c_y，失真参数）并将它们应用到统一摄像机 .
这是我错误的地方 . 我创建一个棋盘图案（6x5），将其内角位置存储为 vector<Point2f> projector2D 并通过投影仪显示它 . kinects现在可以看到棋盘图案，并可以使用 findChessboardCorners(...) 检测其内角，并将它们存储在 vector<Point2f> kinect2D 中 . 校准了kinects颜色和深度流，因此我可以获得相对于每个内角的kinect的真实世界3D位置（ vector<Point3f> kinect3D = kinect->getXYZ(kinect2D) ） . 我甚至以不同的姿势在 table 上举起一张大牌，因此3D位置并非都在同一个平面上（2D投影棋盘保持不变） .
从这个2D投影仪< - > 3D真实世界的映射，我应该能够使用opencv的 calibrateCamera(...) 来计算投影仪的内在函数和外在函数 . extrinsics似乎有意义（kinect大约在 table 上方1米处，投影仪大约比kinect大1米）但重投影错误很大（> 100） . 我还可以做些什么？
我甚至将3D点从kinect的坐标系转换为表的坐标系（所以z = 0），这样我就不必猜测相机的内部参数了 . 现在重投影错误较小，但内在函数现在根本没有意义（c_y应该大致为resolutionHeight / 2） .
这是我的代码：
bool FlowPut::calibrateProjector()
{
// create fullscreen window on second screen
namedWindow("projector", CV_WINDOW_NORMAL);
moveWindow("projector", -1920, 0);
setWindowProperty("projector", CV_WND_PROP_FULLSCREEN, CV_WINDOW_FULLSCREEN);
// construct chessboard pattern image & store inner corner positions
Size patternSize(6, 5);
int width = 1920;
int height = 1080;
int marginX = 0.1f * width;
int marginY = 0.1f * height;
int widthPerCell = (width - 2*marginX) / (patternSize.width + 1);
int heightPerCell = (height - 2*marginY) / (patternSize.height + 1);
Mat chessBoardPattern(height, width, CV_8UC1, Scalar(255));
vector<Point2f> cornersInProjector2D;
for (int r = 0; r < patternSize.height + 1; ++r) {
for (int c = 0; c < patternSize.width + 1; ++c) {
int x = marginX + c*widthPerCell;
int y = marginY + r*heightPerCell;
unsigned char color = 255 * ((c + r*(patternSize.width+1)) % 2);
Rect roi(x, y, widthPerCell, heightPerCell);
chessBoardPattern(roi).setTo(Scalar::all(color));
if (r > 0 && c > 0)
cornersInProjector2D.push_back(Point2f(x, y));
}
}
imshow("projector", chessBoardPattern);
waitKey(1);
// acquire calibration pairs (projector 2d to kinect 3d) from multiple views
int numViewsNeeded = 1;
Mat colorFrame, colorFrameGray, depthFrame;
vector<vector<Point3f> > objectPoints;
vector<vector<Point2f> > imagePoints;
vector<Eigen::Affine3f> transforms;
while (objectPoints.size() < numViewsNeeded) {
inputSource_->updateFrame();
inputSource_->getColorFrameRegistered(colorFrame);
inputSource_->getDepthFrame(depthFrame);
imshow("color", colorFrame);
imshow("depth", depthFrame);
if (waitKey(1) == 112) { // "p"
vector<Point2f> imagePointsLocal;
vector<Point3f> objectPointsLocal;
// find corners2D in kinect image
vector<Point2f> cornersInKinect2D;
cvtColor(colorFrame, colorFrameGray, CV_BGR2GRAY);
if (!findChessboardCorners(colorFrameGray, patternSize, cornersInKinect2D, CV_CALIB_CB_ADAPTIVE_THRESH | CV_CALIB_CB_FILTER_QUADS))
continue;
cornerSubPix(colorFrameGray, cornersInKinect2D, Size(5, 5), Size(-1, -1), TermCriteria(CV_TERMCRIT_EPS + CV_TERMCRIT_ITER, 30, 0.1));
// calculate 3d positions
for (int i = 0; i < cornersInKinect2D.size(); ++i) {
Point2f pProjector2D = cornersInProjector2D[i];
Point2f pKinect2D = cornersInKinect2D[i];
Point3f pKinect3D = inputSource_->getXYZ(pKinect2D.x, pKinect2D.y);
if (isfinite(pKinect3D.x)) {
imagePointsLocal.push_back(pProjector2D);
objectPointsLocal.push_back(pKinect3D);
}
}
// transform 3D positions to their own coordinate system
PointCloud<PointXYZ>::Ptr cloud(new PointCloud<PointXYZ>());
for (const Point3f &p : objectPointsLocal)
cloud->push_back(PointXYZ(p.x, p.y, p.z));
Eigen::Affine3f transform;
PCLHelper::computePCATransform(cloud, transform, -Eigen::Vector3f::UnitZ());
transformPointCloud(*cloud, *cloud, transform);
for (int i = 0; i < imagePointsLocal.size(); ++i) {
const PointXYZ &p = cloud->points[i];
Point3f newP(p.x, p.y, p.z);
cout << "projector2D: " << imagePointsLocal[i] << " kinect3D: " << objectPointsLocal[i] << " table3D: " << newP << endl;
newP.z = 0;
objectPointsLocal[i] = newP;
}
// display
drawChessboardCorners(colorFrame, patternSize, Mat(cornersInKinect2D), true);
imshow("calib_" + to_string(objectPoints.size()), colorFrame);
// store
objectPoints.push_back(objectPointsLocal);
imagePoints.push_back(imagePointsLocal);
transforms.push_back(transform);
}
}
Mat cameraMatrix = Mat::eye(3, 3, CV_64F);
cameraMatrix.at<double>(0, 0) = 1; // wild guess
cameraMatrix.at<double>(1, 1) = 1; // wild guess
cameraMatrix.at<double>(0, 2) = 0.5 * width - 0.5;
cameraMatrix.at<double>(1, 2) = 0.5 * height - 0.5;
cameraMatrix.at<double>(2, 2) = 1.0;
Mat distCoeffs = Mat::zeros(8, 1, CV_64F);
vector<Mat> rvecs, tvecs;
int flags = 0;
//flags = CV_CALIB_USE_INTRINSIC_GUESS;
//flags |= CV_CALIB_FIX_ASPECT_RATIO;
//flags |= CV_CALIB_FIX_K1 | CV_CALIB_FIX_K2 | CV_CALIB_FIX_K3;
//flags |= CV_CALIB_ZERO_TANGENT_DIST;
//flags |= CV_CALIB_FIX_PRINCIPAL_POINT;
double projectionError = calibrateCamera(objectPoints, imagePoints, Size(width, height), cameraMatrix, distCoeffs, rvecs, tvecs, flags);
cout << "projectionError: " << projectionError << endl;
cout << "cameraMatrix" << endl << cameraMatrix << endl;
cout << "distCoeffs" << endl << distCoeffs << endl;
cout << "translations" << endl;
for (const Mat &tvec : tvecs)
cout << tvec << endl;
cout << "rotations" << endl;
for (const Mat &rvec : rvecs) {
Mat rMat;
Rodrigues(rvec, rMat);
cout << rMat << endl;
}
cout << "transforms" << endl;
for (const Eigen::Affine3f aff : transforms)
cout << aff.matrix() << endl;
destroyWindow("projector");
return true;
}
这是输出：
projector2D: [411, 252] kinect3D: [0.395977, 0.127067, 0.949389] table3D: [0.341782, -0.172719, 0.00355369]
projector2D: [630, 252] kinect3D: [0.265477, 0.0954791, 0.964878] table3D: [0.206679, -0.173131, -0.00032872]
projector2D: [849, 252] kinect3D: [0.134238, 0.062928, 0.976424] table3D: [0.070973, -0.173593, -0.000131011]
projector2D: [1068, 252] kinect3D: [0.00167598, 0.0304205, 0.988115] table3D: [-0.0660144, -0.174406, -3.76701e-005]
projector2D: [1287, 252] kinect3D: [-0.132276, -0.0027268, 0.999953] table3D: [-0.204513, -0.17494, 9.65595e-005]
projector2D: [1506, 252] kinect3D: [-0.267018, -0.0363841, 1.01321] table3D: [-0.344021, -0.174915, -0.00101644]
projector2D: [411, 396] kinect3D: [0.416253, 0.0429084, 0.969855] table3D: [0.339156, -0.0838518, 0.000656247]
projector2D: [630, 396] kinect3D: [0.285356, 0.0111659, 0.981753] table3D: [0.203945, -0.0849229, 0.000327408]
projector2D: [849, 396] kinect3D: [0.154899, -0.0207928, 0.995246] table3D: [0.0689683, -0.0853642, -0.00152922]
projector2D: [1068, 396] kinect3D: [0.0232888, -0.052793, 1.0063] table3D: [-0.0669203, -0.0865392, -0.000952244]
projector2D: [1287, 396] kinect3D: [-0.107317, -0.0865693, 1.01591] table3D: [-0.202145, -0.0860579, 0.00137049]
projector2D: [1506, 396] kinect3D: [-0.240572, -0.120209, 1.02912] table3D: [-0.340208, -0.0856784, 0.000245035]
projector2D: [411, 540] kinect3D: [0.436322, -0.0395136, 0.989427] table3D: [0.33683, 0.0031431, -0.00172836]
projector2D: [630, 540] kinect3D: [0.306071, -0.0707654, 0.99843] table3D: [0.202615, 0.00120574, 0.000643492]
projector2D: [849, 540] kinect3D: [0.17609, -0.102898, 1.01177] table3D: [0.0680687, 0.00102064, -0.00104523]
projector2D: [1068, 540] kinect3D: [0.0463136, -0.136332, 1.02396] table3D: [-0.0664956, 0.00189427, -0.00133801]
projector2D: [1287, 540] kinect3D: [-0.0848548, -0.16929, 1.03384] table3D: [-0.202087, 0.00150889, 0.00057447]
projector2D: [1506, 540] kinect3D: [-0.216815, -0.202081, 1.04565] table3D: [-0.338572, 0.00114283, 0.000584126]
projector2D: [411, 684] kinect3D: [0.454245, -0.120754, 1.00522] table3D: [0.333046, 0.0877317, -0.00059998]
projector2D: [630, 684] kinect3D: [0.326484, -0.152412, 1.01794] table3D: [0.200814, 0.0875426, -0.00185949]
projector2D: [849, 684] kinect3D: [0.196626, -0.183715, 1.02961] table3D: [0.0667335, 0.0862762, -0.00209552]
projector2D: [1068, 684] kinect3D: [0.0686593, -0.217207, 1.04014] table3D: [-0.0659516, 0.0873433, -0.000820816]
projector2D: [1287, 684] kinect3D: [-0.0621101, -0.249929, 1.05224] table3D: [-0.201294, 0.0872703, -0.00113678]
projector2D: [1506, 684] kinect3D: [-0.191983, -0.282971, 1.06092] table3D: [-0.335551, 0.0870624, 0.00190979]
projector2D: [411, 828] kinect3D: [0.471262, -0.199975, 1.01816] table3D: [0.329126, 0.169617, 0.00292253]
projector2D: [630, 828] kinect3D: [0.344672, -0.231911, 1.0313] table3D: [0.197922, 0.170072, 0.00127441]
projector2D: [849, 828] kinect3D: [0.217777, -0.263573, 1.04313] table3D: [0.0666032, 0.169934, 0.000858486]
projector2D: [1068, 828] kinect3D: [0.0893882, -0.296656, 1.05822] table3D: [-0.0667883, 0.171399, -0.0023976]
projector2D: [1287, 828] kinect3D: [-0.0399663, -0.329776, 1.06995] table3D: [-0.200828, 0.171994, -0.00232649]
projector2D: [1506, 828] kinect3D: [-0.167329, -0.361072, 1.07448] table3D: [-0.331874, 0.169959, 0.00432938]
projectionError: 1.19223
cameraMatrix
[2019.16764855345, 0, 948.0981380429103;
0, 2074.876304934358, 370.5848700207086;
0, 0, 1]
distCoeffs
[0.1297589355520183;
-3.511519048733476;
-0.005197028270677967;
-0.012067317877384;
23.18698415487345]
translations
[0.00771433482275145;
0.1003592362542408;
1.242695870159661]
rotations
[-0.9999443491624133, -0.003817450239401587, -0.009834920021401414;
-0.002546646669596347, 0.9920108573608824, -0.1261268150270501;
0.01023783028272609, -0.1260947498978334, -0.9919653728227135]
transforms
0.966158 0.242777 -0.0871718 0.0111167
0.25557 -0.946741 0.195871 -0.339577
-0.0349761 -0.211521 -0.976747 0.971594
0 0 0 1
即使我对投影机进行了正确的校准，我如何在Unity中设置这些值，还是在没有Unity的情况下有更好的解决方案？任何帮助都非常感谢！