diff --git a/src/gallium/auxiliary/tgsi/tgsi_strings.c b/src/gallium/auxiliary/tgsi/tgsi_strings.c
index ce51306..6e80011 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_strings.c
+++ b/src/gallium/auxiliary/tgsi/tgsi_strings.c
@@ -58,6 +58,7 @@ static const char *tgsi_file_names[] =
    "SVIEW",
    "BUFFER",
    "MEMORY",
+   "HWATOMIC",
 };
 
 const char *tgsi_semantic_names[TGSI_SEMANTIC_COUNT] =
diff --git a/src/gallium/auxiliary/tgsi/tgsi_ureg.c b/src/gallium/auxiliary/tgsi/tgsi_ureg.c
index 126f096..ded78c2 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_ureg.c
+++ b/src/gallium/auxiliary/tgsi/tgsi_ureg.c
@@ -76,6 +76,7 @@ struct ureg_tokens {
 #define UREG_MAX_SYSTEM_VALUE PIPE_MAX_ATTRIBS
 #define UREG_MAX_OUTPUT PIPE_MAX_SHADER_OUTPUTS
 #define UREG_MAX_CONSTANT_RANGE 32
+#define UREG_MAX_HW_ATOMIC_RANGE 32
 #define UREG_MAX_IMMEDIATE 4096
 #define UREG_MAX_ADDR 3
 #define UREG_MAX_PRED 1
@@ -89,6 +90,15 @@ struct const_decl {
    unsigned nr_constant_ranges;
 };
 
+struct hw_atomic_decl {
+   struct {
+      unsigned first;
+      unsigned last;
+      unsigned array_id;
+   } hw_atomic_range[UREG_MAX_HW_ATOMIC_RANGE];
+   unsigned nr_hw_atomic_ranges;
+};
+
 #define DOMAIN_DECL 0
 #define DOMAIN_INSN 1
 
@@ -164,6 +174,8 @@ struct ureg_program
    struct const_decl const_decls;
    struct const_decl const_decls2D[PIPE_MAX_CONSTANT_BUFFERS];
 
+   struct hw_atomic_decl hw_atomic_decls[PIPE_MAX_HW_ATOMIC_BUFFERS];
+
    unsigned properties[TGSI_PROPERTY_COUNT];
 
    unsigned nr_addrs;
@@ -490,6 +502,30 @@ out:
    return ureg_src_register(TGSI_FILE_CONSTANT, index);
 }
 
+
+/* Returns a new hw atomic register.  Keep track of which have been
+ * referred to so that we can emit decls later.
+ */
+void
+ureg_DECL_hw_atomic(struct ureg_program *ureg,
+                    unsigned first,
+                    unsigned last,
+                    unsigned buffer_id,
+                    unsigned array_id)
+{
+   struct hw_atomic_decl *decl = &ureg->hw_atomic_decls[buffer_id];
+
+   if (decl->nr_hw_atomic_ranges < UREG_MAX_HW_ATOMIC_RANGE) {
+      uint i = decl->nr_hw_atomic_ranges++;
+
+      decl->hw_atomic_range[i].first = first;
+      decl->hw_atomic_range[i].last = last;
+      decl->hw_atomic_range[i].array_id = array_id;
+   } else {
+      set_bad(ureg);
+   }
+}
+
 static struct ureg_dst alloc_temporary( struct ureg_program *ureg,
                                         boolean local )
 {
@@ -1336,6 +1372,35 @@ emit_property(struct ureg_program *ureg,
    out[1].prop_data.Data = data;
 }
 
+static void
+emit_decl_atomic_2d(struct ureg_program *ureg,
+                    unsigned first,
+                    unsigned last,
+                    unsigned index2D,
+                    unsigned array_id)
+{
+   union tgsi_any_token *out = get_tokens(ureg, DOMAIN_DECL, array_id ? 4 : 3);
+
+   out[0].value = 0;
+   out[0].decl.Type = TGSI_TOKEN_TYPE_DECLARATION;
+   out[0].decl.NrTokens = 3;
+   out[0].decl.File = TGSI_FILE_HW_ATOMIC;
+   out[0].decl.UsageMask = TGSI_WRITEMASK_XYZW;
+   out[0].decl.Dimension = 1;
+   out[0].decl.Array = array_id != 0;
+
+   out[1].value = 0;
+   out[1].decl_range.First = first;
+   out[1].decl_range.Last = last;
+
+   out[2].value = 0;
+   out[2].decl_dim.Index2D = index2D;
+
+   if (array_id) {
+      out[3].value = 0;
+      out[3].array.ArrayID = array_id;
+   }
+}
 
 static void emit_decls( struct ureg_program *ureg )
 {
@@ -1432,6 +1497,22 @@ static void emit_decls( struct ureg_program *ureg )
       }
    }
 
+   for (i = 0; i < PIPE_MAX_HW_ATOMIC_BUFFERS; i++) {
+      struct hw_atomic_decl *decl = &ureg->hw_atomic_decls[i];
+
+      if (decl->nr_hw_atomic_ranges) {
+         uint j;
+
+         for (j = 0; j < decl->nr_hw_atomic_ranges; j++) {
+            emit_decl_atomic_2d(ureg,
+                                decl->hw_atomic_range[j].first,
+                                decl->hw_atomic_range[j].last,
+                                i,
+                                decl->hw_atomic_range[j].array_id);
+         }
+      }
+   }
+
    if (ureg->nr_temps) {
       unsigned array = 0;
       for (i = 0; i < ureg->nr_temps;) {
@@ -1483,7 +1564,6 @@ static void copy_instructions( struct ureg_program *ureg )
           nr_tokens * sizeof out[0] );
 }
 
-
 static void
 fixup_header_size(struct ureg_program *ureg)
 {
diff --git a/src/gallium/auxiliary/tgsi/tgsi_ureg.h b/src/gallium/auxiliary/tgsi/tgsi_ureg.h
index 65aedd9..a68af19 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_ureg.h
+++ b/src/gallium/auxiliary/tgsi/tgsi_ureg.h
@@ -253,6 +253,13 @@ struct ureg_src
 ureg_DECL_constant( struct ureg_program *,
                     unsigned index );
 
+void
+ureg_DECL_hw_atomic(struct ureg_program *ureg,
+                    unsigned first,
+                    unsigned last,
+                    unsigned buffer_id,
+                    unsigned array_id);
+
 struct ureg_dst
 ureg_DECL_temporary( struct ureg_program * );
 
diff --git a/src/gallium/include/pipe/p_shader_tokens.h b/src/gallium/include/pipe/p_shader_tokens.h
index 94fb929..acfbeb5 100644
--- a/src/gallium/include/pipe/p_shader_tokens.h
+++ b/src/gallium/include/pipe/p_shader_tokens.h
@@ -80,6 +80,7 @@ enum tgsi_file_type {
    TGSI_FILE_SAMPLER_VIEW        =11,
    TGSI_FILE_BUFFER,
    TGSI_FILE_MEMORY,
+   TGSI_FILE_HW_ATOMIC,
    TGSI_FILE_COUNT      /**< how many TGSI_FILE_ types */
 };
 
diff --git a/src/gallium/include/pipe/p_state.h b/src/gallium/include/pipe/p_state.h
index 45d9cb7..dfeec40 100644
--- a/src/gallium/include/pipe/p_state.h
+++ b/src/gallium/include/pipe/p_state.h
@@ -69,7 +69,7 @@ extern "C" {
 #define PIPE_MAX_VIEWPORTS        16
 #define PIPE_MAX_CLIP_OR_CULL_DISTANCE_COUNT 8
 #define PIPE_MAX_CLIP_OR_CULL_DISTANCE_ELEMENT_COUNT 2
-
+#define PIPE_MAX_HW_ATOMIC_BUFFERS 32
 
 struct pipe_reference
 {