stencil buffer - Correctly use stencil_texturing in OpenGL -

i trying implement stencil_texturing extension of opengl proof of concept. video card supports gl 4.3 stencil_texturing available me. if more clarification necessary here spec provided: http://www.opengl.org/registry/specs/arb/stencil_texturing.txt.

so goal of test render color buffer texture in frame 0, depth buffer in frame 1 , stencil buffer in frame 2. easy part done , have color , depth buffer textures rendered fine. issue lies stencil buffer , believe issue coming either lack of understanding stencil buffers (which case) or misuse of stencil_texturing. tried find info online there little available.

to give idea of rendering here current frame captures: color buffer, depth buffer, stencil buffer

so vision stencil buffer stencil out middle triangle, in middle triangle has value of 1 , every part of texture has value of 0. not sure how come when rendering imagine areas stencil value of 1 different 0.

here code below. test class throw framework made them. believe thing not definied glerr() calls glgeterror() make sure correct.

   typedef struct    {      gluint program;      gluint vshader;      gluint fshader;    } stenciltexturingstate;     class teststenciltexturing : public testinfo    {     public:     teststenciltexturing(testconfig& config, int argc, char** argv)          :width(config.windowwidth), height(config.windowheight)     {          state = (stenciltexturingstate*) malloc(sizeof(stenciltexturingstate));     }      ~teststenciltexturing()      {         destroyteststenciltexturing();     }      void loadfboshaders()     {         const char* vshader = "assets/stencil_texturing/fbo_vert.vs";         const char* fshader = "assets/stencil_texturing/fbo_frag.fs";          state->vshader = loadshader(vshader, gl_vertex_shader);         glerr();         state->fshader = loadshader(fshader, gl_fragment_shader);         glerr();         state->program = link(state->vshader, state->fshader, 1, "inposition");         glerr();          gluseprogram(state->program);     }      void loadtextureshaders()     {         const char* vshader = "assets/stencil_texturing/tex_vert.vs";         const char* fshader = "assets/stencil_texturing/tex_frag.fs";          state->vshader = loadshader(vshader, gl_vertex_shader);         glerr();         state->fshader = loadshader(fshader, gl_fragment_shader);         glerr();         state->program = link(state->vshader, state->fshader, 1, "inposition");         glerr();          gluseprogram(state->program);     }      void destroyteststenciltexturing()     {         gluseprogram(0);         gldeleteshader(state->vshader);         gldeleteshader(state->fshader);         gldeleteprogram(state->program);         free(state);     }      void rendertotexture(gluint renderedtexture, int frame)     {         glint  posid, colid;         gluint fboid, depth_stencil_rb;          const float vertexfbopositions[] =          {             -0.7f, -0.7f,  0.5f,  1.0f,             0.7f,  -0.7f,  0.5f,  1.0f,             0.6f,  0.7f,   0.5f,  1.0f,         };          const float vertexfbocolors[] =          {             1.0f, 1.0f, 0.0f, 1.0f,             1.0f, 1.0f, 1.0f, 1.0f,             1.0f, 0.0f, 0.0f, 1.0f,         };          // load shaders fbo         loadfboshaders();          // setup fbo         glgenframebuffers(1, &fboid);         glbindframebuffer(gl_framebuffer, fboid);         glviewport(0, 0, width, height);          // set renderbuffer depth_stencil formats.         glgenrenderbuffers(1, &depth_stencil_rb);         glbindrenderbuffer(gl_renderbuffer, depth_stencil_rb);         glrenderbufferstorage(gl_renderbuffer, gl_depth24_stencil8, width, height);         glframebufferrenderbuffer(gl_framebuffer, gl_depth_stencil_attachment,                                    gl_renderbuffer, depth_stencil_rb);          // depending on frame bind 2d texture differently.         // frame 0 - color, frame 1 - depth, frame 2 - stencil         glbindtexture(gl_texture_2d, renderedtexture);         gltexparameteri(gl_texture_2d, gl_texture_wrap_s, gl_clamp_to_edge);         gltexparameteri(gl_texture_2d, gl_texture_wrap_t, gl_clamp_to_edge);         gltexparameteri(gl_texture_2d, gl_texture_mag_filter, gl_nearest);         gltexparameteri(gl_texture_2d, gl_texture_min_filter, gl_nearest);          // create our rgba texture render our color buffer into.         if (frame == 0)         {             glteximage2d(gl_texture_2d, 0, gl_rgba, width, height, 0, gl_rgba, gl_unsigned_byte, null);             glframebuffertexture2d(gl_framebuffer, gl_color_attachment0, gl_texture_2d, renderedtexture, 0);         }          // create our depth24_stencil8 texture render our depth buffer into.         if (frame == 1)         {             glenable(gl_depth_test);               glteximage2d(gl_texture_2d, 0, gl_depth24_stencil8, width, height, 0, gl_depth_stencil, gl_unsigned_int_24_8, null);             glframebuffertexture2d(gl_framebuffer, gl_depth_stencil_attachment, gl_texture_2d, renderedtexture, 0);          }          // create our depth24_stencil8 texture , change depth_stencil_texture mode         // render our stencil buffer into.         if (frame == 2)         {             glenable(gl_depth_test | gl_stencil_test);              glteximage2d(gl_texture_2d, 0, gl_depth24_stencil8, width, height, 0, gl_depth_stencil, gl_unsigned_int_24_8, null);             glframebuffertexture2d(gl_framebuffer, gl_depth_stencil_attachment, gl_texture_2d, renderedtexture, 0);              gltexparameteri(gl_texture_2d, gl_depth_stencil_texture_mode, gl_stencil_index);         }          glerr();          glenum status = glcheckframebufferstatus(gl_framebuffer);         if (status != gl_framebuffer_complete)         {             printf("there error framebuffer, fix it!\n");         }         glerr();          // give values of position , color of our triangle shaders.         posid = glgetattriblocation(state->program, "position");         colid = glgetattriblocation(state->program, "color");         glerr();          glvertexattribpointer(posid, 4, gl_float, 0, 0, vertexfbopositions);         glenablevertexattribarray(posid);         glvertexattribpointer(colid, 4, gl_float, 0, 0, vertexfbocolors);         glenablevertexattribarray(colid);          // clear depth buffer 1.0f draw our rgb stripes far back.         glcleardepth(1.0f);         glclear(gl_depth_buffer_bit);          if (frame == 2)         {             glstencilfunc(gl_never, 1, 0xff); // never pass stencil test             glstencilop(gl_replace, gl_keep, gl_keep);  // replace stencil buffer values ref=1             glstencilmask(0xff); // stencil buffer free write             glclear(gl_stencil_buffer_bit);  // first clear stencil buffer writing default stencil value (0) of stencil buffer.             gldrawarrays(gl_triangles, 0, 3); // @ stencil shape pixel locations in stencil buffer replace stencil buffer values ref = 1              // no more modifying of stencil buffer on stencil , depth pass.             glstencilmask(0x00);             // can achieved glstencilop(gl_keep, gl_keep, gl_keep);              // stencil test: pass stencil test @ stencilvalue == 1 (assuming depth test pass.) , write actual content depth , color buffer @ stencil shape locations.             glstencilfunc(gl_equal, 1, 0xff);         }          // use scissors clear fbo rgb stripped pattern.         glenable(gl_scissor_test);         glscissor(width * 0/3, 0, width * 1/3, height);         glclearcolor(0.54321f, 0.0f, 0.0f, 0.54321f); // red         glclear(gl_color_buffer_bit);         glscissor(width * 1/3, 0, width * 2/3, height);         glclearcolor(0.0f, 0.65432f, 0.0f, 0.65432f); // green         glclear(gl_color_buffer_bit);         glscissor(width * 2/3, 0, width * 3/3, height);         glclearcolor(0.0f, 0.0f, 0.98765f, 0.98765f); // blue         glclear(gl_color_buffer_bit);         gldisable(gl_scissor_test);         glerr();          gldrawarrays(gl_triangles, 0, 3);          gldisable(gl_depth_test);          glerr();          // remove fbo , shaders , return original viewport.         gluseprogram(0);         glbindframebuffer(gl_framebuffer, 0);         gldeleteshader(state->vshader);         gldeleteshader(state->fshader);         gldeleteprogram(state->program);         gldeleteframebuffers(1, &fboid);         glviewport(0, 0, width, height);         glerr();     }      void drawframeteststenciltexturing(int frame)     {         glint  posloc, texloc;         gluint renderedtexture;          const glubyte indxbuf[] = {0, 1, 2, 1, 3, 2};          const float positions[] =          {             -0.8f, -0.8f,             -0.8f,  0.8f,             0.8f, -0.8f,             0.8f, 0.8f,         };          const float texcoords[] =          {             0.0f, 0.0f,             0.0f, 1.0f,             1.0f, 0.0f,             1.0f, 1.0f         };          // allocate , initialize texture rendered to, ,         // textured onto quad on default framebuffer.         glgentextures(1, &renderedtexture);          // render texture using fbo.         rendertotexture(renderedtexture, frame);          // create , load shaders draw texture.         loadtextureshaders();          // draw texture window.         glclearcolor(0.25f, 0.25f, 0.25f, 1.0f);         glclear(gl_color_buffer_bit);          posloc = glgetattriblocation(state->program, "position");         texloc = glgetattriblocation(state->program, "a_texcoords");          glvertexattribpointer(posloc, 2, gl_float, 0, 0, positions);         glenablevertexattribarray(posloc);         glvertexattribpointer(texloc, 2, gl_float, 0, 0, texcoords);         glenablevertexattribarray(texloc);          // draw our generated texture onto quad.         gldrawelements(gl_triangles, 6, gl_unsigned_byte, indxbuf);         glflush();          gldeletetextures(1, &renderedtexture);         glerr();     }      void rendertest(int frame)     {            drawframeteststenciltexturing(frame);     }  private:     stenciltexturingstate* state;     const int height, width; };  run_test(stenciltexturing, "stencil_texturing", 2); 

the line

glenable(gl_depth_test | gl_stencil_test); 

is not going work, gl enable enums not bit values, enums, might enable else, or gl_invalid_enum error, don't enable stencil test here.