qcow2_helper: support extended_l2 feature

Signed-off-by: Anthoine Bourgeois <anthoine.bourgeois@vates.tech>

Author: AnthoineB

qcow2/qcow2_helper.c

@@ -21,6 +21,8 @@ static void transform_header_be_to_le(struct qcow2_header* header){
     SWAP_BE_TO_LE(32, header_length);
 }
 
+//#define DEBUG
+#ifdef DEBUG
 char* qcow2_get_backing_file(struct qcow2_header* header, int fd){
     int err, backing_file_name_size;
     char* backing_file_name;
@@ -43,6 +45,7 @@ char* qcow2_get_backing_file(struct qcow2_header* header, int fd){
     }
     return NULL;
 }
+#endif
 
 uint64_t* get_l1_offset(struct qcow2_header* header, int fd){
     int i, err = 0;
@@ -71,12 +74,11 @@ uint64_t* get_l1_offset(struct qcow2_header* header, int fd){
     return raw_l1;
 }
 
-uint64_t* get_l2_table(struct qcow2_header* header, int fd, uint64_t offset){
+uint64_t* get_l2_table(struct qcow2_header* header, int fd, uint64_t offset, uint64_t nb_l2_entries, int extended_l2){
     int i;
     ssize_t bytes_read;
     uint64_t* raw_l2 = NULL;
     uint64_t cluster_size = (1 << header->cluster_bits);
-    uint64_t nb_l2_entries = (cluster_size / sizeof(uint64_t));
 
     raw_l2 = malloc(cluster_size);
     if(raw_l2 == NULL){
@@ -92,7 +94,7 @@ uint64_t* get_l2_table(struct qcow2_header* header, int fd, uint64_t offset){
         return NULL;
     }
 
-    for(i = 0; i < nb_l2_entries; i++){
+    for(i = 0; i < nb_l2_entries * (extended_l2 ? 2 : 1); i++){
         raw_l2[i] = __builtin_bswap64(raw_l2[i]);
     }
 
@@ -107,7 +109,57 @@ int is_l2_allocated(uint64_t l2_entry){
     return ((l2_entry & ALLOCATED_ENTRY_BIT) != 0) || ((l2_entry & STANDARD_CLUSTER_OFFSET_MASK) != 0);
 }
 
-uint64_t get_cluster_to_byte(uint64_t allocated_clusters, uint64_t cluster_size){
+int is_extended_l2_allocated(uint64_t l2_entry_lo, uint64_t l2_entry_hi){
+    if((l2_entry_lo & CLUSTER_TYPE_BIT) != 0){
+        fprintf(stderr, "Cluster is compressed\n");
+        exit(EXIT_FAILURE); //TODO: Read compressed clusters
+    }
+    return l2_entry_hi & 0xffffffff;
+}
+
+uint32_t count_set_bits(uint32_t alloc_status_bitmap)
+{
+    uint32_t count = 0;
+
+    if (alloc_status_bitmap == 0)
+        return 0;
+
+    while (alloc_status_bitmap) {
+        alloc_status_bitmap &= alloc_status_bitmap - 1;
+        count++;
+    }
+
+    return count;
+}
+
+uint32_t get_extended_l2_allocated(uint64_t l2_entry_lo, uint64_t l2_entry_hi){
+    if((l2_entry_lo & CLUSTER_TYPE_BIT) != 0){
+        fprintf(stderr, "Cluster is compressed\n");
+        exit(EXIT_FAILURE); //TODO: Read compressed clusters
+    }
+    return count_set_bits(l2_entry_hi & 0xffffffff);
+}
+
+uint64_t get_allocated_clusters(uint64_t nb_l2_entries, uint64_t *l2_table, int extended_l2)
+{
+    uint64_t allocated_clusters = 0;
+    int j;
+    for(j = 0; j < nb_l2_entries; j++){
+        if (extended_l2) {
+            allocated_clusters += get_extended_l2_allocated(l2_table[j*2], l2_table[j*2+1]);
+        } else {
+            if(is_l2_allocated(l2_table[j])){
+                allocated_clusters += 1;
+            }
+        }
+    }
+    return allocated_clusters;
+}
+
+uint64_t get_cluster_to_byte(uint64_t allocated_clusters, uint64_t cluster_size, int extended_l2){
+    if (extended_l2) {
+        return allocated_clusters * (cluster_size / 32);
+    }
     return allocated_clusters * cluster_size;
 }
 
@@ -119,14 +171,35 @@ void set_bit(char* m, int bit, int val){
     *m |= (val << bit);
 }
 
-void dump_bitmap(struct qcow2_header* header, int fd, uint64_t *l1_table){
+void set_l1_bitmap(char *base_l1_bitmap, uint64_t *l2_table, uint64_t nb_l2_entries, int extended_l2) {
+    int j;
+    int mementry;
+    int bit;
+
+    for (j = 0; j < nb_l2_entries; j++) {
+        if (extended_l2) {
+            if(!is_extended_l2_allocated(l2_table[j*2], l2_table[j*2+1])) {
+                continue;
+            }
+        } else {
+            if(!is_l2_allocated(l2_table[j])){
+                continue;
+            }
+        }
+        mementry = j/8;
+        bit = j%8;
+        //Mark L2 entry allocated
+        set_bit(&(base_l1_bitmap[mementry]), bit, 1);
+    }
+}
+
+void dump_bitmap(struct qcow2_header* header, int fd, uint64_t *l1_table, uint64_t nb_l2_entries, int extended_l2){
     int i, n;
     char* bitmap = NULL;
     uint64_t cluster_size = (1 << header->cluster_bits); //cluster size in bytes
     uint64_t total_blocks, bitmap_size;
-    uint64_t nb_l2_entries = (cluster_size / sizeof(uint64_t)); //Number of L2 in a L1 entry
 
-	total_blocks = header->size / cluster_size;
+    total_blocks = header->size / cluster_size;
     bitmap_size = total_blocks >> 3; // This transform our number of bits in a number of bytes for allocation
     //Does VHD use sectors of 512 for the bitmap it dumps? Nope, it uses 2MiB. Do we want to use 2MiB to reduce QCOW2 size allocation? We would need a way to transform x 64KiB blocks in a 2MiB block.
     bitmap = malloc(bitmap_size);
@@ -135,25 +208,16 @@ void dump_bitmap(struct qcow2_header* header, int fd, uint64_t *l1_table){
 
     #pragma omp parallel for num_threads(4)
     for(i = 0; i < header->l1_size; i++){
-        int j;
         uint64_t *l2_table = NULL;
         uint64_t l1_entry = l1_table[i];
         if(l1_entry != 0){
-            l2_table = get_l2_table(header, fd, l1_entry); if(l2_table == NULL) { fprintf(stderr, "Couldn't get L2 entry\n"); exit(EXIT_FAILURE); }
-            char* base_l1_bitmap = bitmap + (i * nb_byte_for_l1);
-            for(j = 0; j < nb_l2_entries; j++){
-                if(is_l2_allocated(l2_table[j])){
-                    //Mark L2 entry allocated
-
-                    int mementry = j/8;
-                    int bit = j%8;
-                    set_bit(&(base_l1_bitmap[mementry]), bit, 1);
-                }
-                else {
-                    // Mark L2 entry as not allocated
-                    mark_l2_unallocated(bitmap, i, j); // The bytes are already zeroed, we don't need to do anything
-                }
+            l2_table = get_l2_table(header, fd, l1_entry, nb_l2_entries, extended_l2);
+            if(l2_table == NULL) {
+                fprintf(stderr, "Couldn't get L2 table\n");
+                exit(EXIT_FAILURE);
             }
+            char* base_l1_bitmap = bitmap + (i * nb_byte_for_l1);
+            set_l1_bitmap(base_l1_bitmap, l2_table, nb_l2_entries, extended_l2);
             free(l2_table);
         }
         else{
@@ -170,26 +234,21 @@ void dump_bitmap(struct qcow2_header* header, int fd, uint64_t *l1_table){
     free(bitmap);
 }
 
-int get_allocated_blocks(struct qcow2_header* header, int fd, uint64_t *l1_table){
+int get_allocated_blocks(struct qcow2_header* header, int fd, uint64_t *l1_table, uint64_t nb_l2_entries, int extended_l2){
     uint64_t allocated_clusters = 0;
-    int i, cluster_size = (1 << header->cluster_bits), nb_l2_entries = cluster_size / (sizeof(uint64_t));
-
+    int i;
 
     #pragma omp parallel for num_threads(4) reduction (+:allocated_clusters)
     for(i = 0; i < header->l1_size; i++){
-        int j;
         uint64_t *l2_table = NULL;
         uint64_t l1_entry = l1_table[i];
         if(l1_entry != 0){
-            l2_table = get_l2_table(header, fd, l1_entry);
+            l2_table = get_l2_table(header, fd, l1_entry, nb_l2_entries, extended_l2);
             if(l2_table == NULL){
                 fprintf(stderr, "Couldn't get L2 Table");
+                exit(EXIT_FAILURE);
             }
-            for(j = 0; j < nb_l2_entries; j++){
-                if(is_l2_allocated(l2_table[j])){
-                    allocated_clusters += 1;
-                }
-            }
+            allocated_clusters += get_allocated_clusters(nb_l2_entries, l2_table, extended_l2);
             free(l2_table);
         }
     }
@@ -200,7 +259,9 @@ int main(int argc, char* argv[]){
     struct qcow2_header* header = NULL;
     char * command, * filename = NULL, * backing_file_name = NULL;
     int fd, err = 0, ret = EXIT_SUCCESS;
+    int extended_l2;
     uint64_t *l1_table = NULL, cluster_size = 0, allocated_clusters = 0, allocated_byte = 0;
+    uint64_t nb_l2_entries;
 
     if(argc != 3){
         fprintf(stderr, "Need an argument\n");
@@ -239,10 +300,20 @@ int main(int argc, char* argv[]){
     }
 
     cluster_size = (1 << header->cluster_bits);
-
-    // printf("Version: %d\n", header->version);
-    // backing_file_name = qcow2_get_backing_file(header, fd);
-    // printf("Backing file: %s\n", backing_file_name);
+    extended_l2 = (header->version == 3) && (header->incompatible_features & INCOMPATIBLE_FEATURE_EXTENDED_L2);
+
+#ifdef DEBUG
+    printf("Version: %d\n", header->version);
+    backing_file_name = qcow2_get_backing_file(header, fd);
+    printf("Backing file: %s\n", backing_file_name);
+    printf("Extended L2: %d\n", extended_l2);
+#endif
+
+    if (extended_l2) {
+        nb_l2_entries = cluster_size / (sizeof(uint64_t) * 2);
+    } else {
+        nb_l2_entries = cluster_size / (sizeof(uint64_t));
+    }
 
     l1_table = get_l1_offset(header, fd);
     if(l1_table == NULL){
@@ -252,11 +323,11 @@ int main(int argc, char* argv[]){
     }
 
     if(!strcmp("bitmap", command)){
-        dump_bitmap(header, fd, l1_table);
+        dump_bitmap(header, fd, l1_table, nb_l2_entries, extended_l2);
     }
     else if(!strcmp("allocated", command)){
-        allocated_clusters = get_allocated_blocks(header, fd, l1_table);
-        allocated_byte = get_cluster_to_byte(allocated_clusters, cluster_size);
+        allocated_clusters = get_allocated_blocks(header, fd, l1_table, nb_l2_entries, extended_l2);
+        allocated_byte = get_cluster_to_byte(allocated_clusters, cluster_size, extended_l2);
         printf("%lu\n", allocated_byte);
     }
     else{

qcow2/qcow_helper.h

@@ -14,6 +14,8 @@
 #define CLUSTER_TYPE_BIT (1UL << 62) /* 0 for standard, 1 for compressed cluster */
 #define ALLOCATED_ENTRY_BIT (1UL << 63) /* Bit 63 is the allocated bit for standard cluster */
 
+/* Incompatible features */
+#define INCOMPATIBLE_FEATURE_EXTENDED_L2 0x0010
 
 struct qcow2_header {
     uint32_t magic;