Leap Motion C API  4.1.0
The API to the LeapC library.
Rectifying with a Shader Example

To rectify an image with a shader, you can upload both the image data and the distortion maps and apply them as textures to an object such as a quad. Instead of sampling the image data directly, your fragment shader instead samples the distortion map texture. You then use the texture coordinates from the distortion texture to look up the brightness value in the image texture. See Rectifying with a Shader for more information about the distortion map texture and the fragment shader to use it.

1 #undef __cplusplus
2 
3 #include <stdio.h>
4 #include <stdlib.h>
5 
6 #ifdef _WIN32
7 #include <Windows.h>
8 #else
9 #include <unistd.h>
10 #endif
11 
12 #include <time.h>
13 #include "LeapC.h"
14 #include "ExampleConnection.h"
15 #include "GLutils.h"
16 #include <math.h>
17 #include <string.h>
18 
19 static const char* GLSL_VERT = STRINGIFY(
20  \n#version 120\n
21 
22  void main()
23  {
24  gl_Position = ftransform();
25  gl_TexCoord[0]=gl_MultiTexCoord0;
26  }
27 );
28 static const char* GLSL_FRAG = STRINGIFY(
29  \n#version 120\n
30  uniform sampler2D rawTexture;
31  uniform sampler2D distortionTexture;
32 
33  void main()
34  {
35  vec2 distortionIndex = texture2D(distortionTexture, gl_TexCoord[0].st).xy;
36  float hIndex = distortionIndex.r;
37  float vIndex = distortionIndex.g;
38 
39  if(vIndex > 0.0 && vIndex < 1.0 && hIndex > 0.0 && hIndex < 1.0)
40  {
41  gl_FragColor = vec4(texture2D(rawTexture, distortionIndex).rrr, 1.0);
42  }
43  else
44  {
45  gl_FragColor = vec4(0.2, 0.0, 0.0, 1.0);
46  }
47  }
48 );
49 
50 GLsizei image_width;
51 GLsizei image_height;
52 void* image_buffer = NULL;
53 float* distortion_buffer_left = NULL;
54 float* distortion_buffer_right = NULL;
55 uint64_t image_size = 0;
56 bool imageReady = false;
57 uint64_t currentDistortionId = 0;
58 
59 int window; // GLUT window handle
60 int windowWidth = 800;
61 int windowHeight = 400;
62 
63 GLuint vertShader, fragShader, program;
64 GLuint rawTextureLeft = 0;
65 GLuint rawTextureRight = 0;
66 GLuint distortionTextureLeft = 0;
67 GLuint distortionTextureRight = 0;
68 
69 /** Callback for when an image is available. */
70 static void OnImage(const LEAP_IMAGE_EVENT *imageEvent){
71  const LEAP_IMAGE_PROPERTIES* properties = &imageEvent->image[0].properties;
72  if (properties->bpp != 1)
73  return;
74 
75  if (properties->width*properties->height != image_size) {
76  void* prev_image_buffer = image_buffer;
77  image_width = properties->width;
78  image_height = properties->height;
79  image_size = image_width*image_height;
80  image_buffer = malloc(2 * image_size);
81  if (prev_image_buffer)
82  free(prev_image_buffer);
83  }
84 
85  memcpy(image_buffer, (char*)imageEvent->image[0].data + imageEvent->image[0].offset, image_size);
86  memcpy((char*)image_buffer + image_size, (char*)imageEvent->image[1].data + imageEvent->image[1].offset, image_size);
87 
88  //Save the distortion data if it's version id changes
89  if(currentDistortionId != imageEvent->image[0].matrix_version){
90  size_t distortion_size = 64 * 64 * 2;
91  distortion_buffer_left = malloc(sizeof(float) * distortion_size);
92  distortion_buffer_right = malloc(sizeof(float) * distortion_size);
93  memcpy(distortion_buffer_left, imageEvent->image[0].distortion_matrix, sizeof(float) * distortion_size);
94  memcpy(distortion_buffer_right, imageEvent->image[1].distortion_matrix, sizeof(float) * distortion_size);
95  currentDistortionId = imageEvent->image[0].matrix_version;
96  }
97  imageReady = true;
98 }
99 
100 static void display(void)
101 {
102  glMatrixMode(GL_MODELVIEW);
103  glLoadIdentity();
104 
105  glPushMatrix();
106  glClearColor(1.0f, 1.0f, 1.0f, 1.0f );
107  glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
108 
109  if(imageReady){
110  //Setup perspective and view matrices
111  glMatrixMode(GL_PROJECTION);
112  checkGLError("set matrix mode");
113  glLoadIdentity();
114  setPerspectiveFrustrum(96, windowWidth/windowHeight, 1, 20);
115  checkGLError("set frustrum");
116 
117  glMatrixMode(GL_MODELVIEW);
118  glPushMatrix();
119  glTranslatef(1.05f, 0.0, -1.01f);
120 
121  glEnable(GL_TEXTURE_2D);
122  glUseProgram(program);
123 
124  // Left camera image
125  glActiveTexture(GL_TEXTURE0);
126  glBindTexture(GL_TEXTURE_2D, rawTextureLeft);
127  glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, image_width, image_height, 0, GL_RED, GL_UNSIGNED_BYTE, image_buffer);
128  checkGLError("Initializing raw texture");
129 
130  glActiveTexture(GL_TEXTURE1);
131  glBindTexture(GL_TEXTURE_2D, distortionTextureLeft);
132  glTexImage2D(GL_TEXTURE_2D, 0, GL_RG, 64, 64, 0, GL_RG, GL_FLOAT, distortion_buffer_left);
133  checkGLError("Initializing distortion texture");
134 
135  glBegin(GL_QUADS); // Draw A Quad for camera image
136  glTexCoord2f(0.0f, 1.0f); glVertex3f(-1.0f, 1.0f, 0.0f); // Top Left
137  glTexCoord2f(1.0f, 1.0f); glVertex3f( 1.0f, 1.0f, 0.0f); // Top Right
138  glTexCoord2f(1.0f, 0.0f); glVertex3f( 1.0f, -1.0f, 0.0f); // Bottom Right
139  glTexCoord2f(0.0f, 0.0f); glVertex3f(-1.0f, -1.0f, 0.0f); // Bottom Left
140  glEnd();
141 
142  // Right camera image
143  glActiveTexture(GL_TEXTURE0);
144  glBindTexture(GL_TEXTURE_2D, rawTextureRight);
145  glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, image_width, image_height, 0, GL_RED, GL_UNSIGNED_BYTE,
146  (GLvoid*)((size_t)(image_buffer) + (image_width * image_height)));
147  checkGLError("Updating raw texture with right image");
148 
149  glActiveTexture(GL_TEXTURE1);
150  glBindTexture(GL_TEXTURE_2D, distortionTextureRight);
151  glTexImage2D(GL_TEXTURE_2D, 0, GL_RG, 64, 64, 0, GL_RG, GL_FLOAT, distortion_buffer_right);
152  checkGLError("Updating distortion texture with right distortion map");
153 
154  glTranslatef(-2.01f, 0.0, 0.0);
155  glBegin(GL_QUADS);
156  glTexCoord2f(0.0f, 1.0f); glVertex3f(-1.0f, 1.0f, 0.0f); // Top Left
157  glTexCoord2f(1.0f, 1.0f); glVertex3f( 1.0f, 1.0f, 0.0f); // Top Right
158  glTexCoord2f(1.0f, 0.0f); glVertex3f( 1.0f,-1.0f, 0.0f); // Bottom Right
159  glTexCoord2f(0.0f, 0.0f); glVertex3f(-1.0f,-1.0f, 0.0f); // Bottom Left
160  glEnd();
161 
162  glPopMatrix();
163 
164  imageReady = false;
165  }
166  glFlush();
167  glPopMatrix();
168  glutSwapBuffers();
169 }
170 
171 static void reshape(int w, int h)
172 {
173  glViewport(0, 0, (GLsizei) w, (GLsizei) h);
174 }
175 
176 static void keyboard(unsigned char key, int x, int y)
177 {
178  switch((char)key) {
179  case 'q':
180  case 27: // ESC
181  glutDestroyWindow(window);
182  CloseConnection();
183  if(image_buffer) free(image_buffer);
184  exit(0);
185  default:
186  break;
187  }
188 }
189 
190 static void idle(void){
191  if(imageReady)
192  glutPostRedisplay();
193 }
194 
195 int main(int argc, char *argv[])
196 {
197  ConnectionCallbacks.on_image = OnImage;
198 
199  LEAP_CONNECTION *connection = OpenConnection();
200  LeapSetPolicyFlags(*connection, eLeapPolicyFlag_Images, 0);
201 
202  while(!IsConnected){
203  millisleep(250);
204  }
205 
206  glutInit(&argc, argv);
207  glutInitDisplayMode(GLUT_DOUBLE | GLUT_RGBA);
208  glutInitWindowSize(windowWidth, windowHeight);
209  window = glutCreateWindow("LeapC Distortion Shader Example");
210 
211 #if defined(GLEW_VERSION)
212  GLenum err = glewInit();
213  if (err != GLEW_OK) {
214  /* Problem: glewInit failed, something is seriously wrong. */
215  fprintf(stderr, "Error: %s\n", glewGetErrorString(err));
216  return 1;
217  }
218 #endif
219 
220  // GLUT callbacks
221  glutIdleFunc(idle);
222  glutReshapeFunc(reshape);
223  glutKeyboardFunc(keyboard);
224  glutDisplayFunc(display);
225 
226  // init GL
227  glClearColor(0.0, 0.0, 0.0, 0.0);
228  glColor3f(1.0, 1.0, 1.0);
229 
230  // init shader and textures
231 
232  //Create the shader program
233  program = createProgram(GLSL_VERT, GLSL_FRAG);
234  glUseProgram(program);
235  GLuint rawSampler = glGetUniformLocation(program, "rawTexture");
236  GLuint distortionSampler = glGetUniformLocation(program, "distortionTexture");
237  glUniform1i(rawSampler, 0);
238  glUniform1i(distortionSampler, 1);
239 
240  //Create textures
241  rawTextureLeft = createTextureReference();
242  distortionTextureLeft = createTextureReference();
243  rawTextureRight = createTextureReference();
244  distortionTextureRight = createTextureReference();
245 
246  // Start GLUT loop
247  glutMainLoop();
248 
249  CloseConnection();
250  free(distortion_buffer_left);
251  free(distortion_buffer_right);
252  return 0;
253 }
254 //End-of-Sample

This example is only supported on platforms for which a working version of GLUT exists. It should not be overly difficult to port the example to a different OpenGL-based context, however.