Check memory allocations for success
https://gitlab.xiph.org/xiph/libao/-/commit/d5221655dfd1a2156aa6be83b5aadea7c1e0f5bd

Index: src/audio_out.c
--- src/audio_out.c.orig
+++ src/audio_out.c
@@ -634,6 +634,10 @@ static char *_sanitize_matrix(int maxchannels, char *m
     char *ret = calloc(strlen(matrix)+1,1); /* can only get smaller */
     char *p=matrix;
     int count=0;
+
+    if(!ret)
+      return NULL;
+
     while(count<maxchannels){
       char *h,*t;
       int m=0;
@@ -707,6 +711,15 @@ static int _find_channel(int needle, char *haystack){
   return -1;
 }
 
+static void _free_map(char **m){
+  char **in=m;
+  while(m && *m){
+    free(*m);
+    m++;
+  }
+  if(in)free(in);
+}
+
 static char **_tokenize_matrix(char *matrix){
   char **ret=NULL;
   char *p=matrix;
@@ -731,6 +744,8 @@ static char **_tokenize_matrix(char *matrix){
   }
 
   ret = calloc(count+1,sizeof(*ret));
+  if(!ret)
+    return NULL;
 
   p=matrix;
   count=0;
@@ -749,6 +764,10 @@ static char **_tokenize_matrix(char *matrix){
     while(t>p && isspace(*(t-1)))t--;
 
     ret[count] = calloc(t-p+1,1);
+    if(!ret[count]){
+      _free_map(ret);
+      return NULL;
+    }
     memcpy(ret[count],p,t-p);
     count++;
     if(!*h)break;
@@ -756,25 +775,22 @@ static char **_tokenize_matrix(char *matrix){
   }
 
   return ret;
-
 }
 
-static void _free_map(char **m){
-  char **in=m;
-  while(m && *m){
-    free(*m);
-    m++;
-  }
-  if(in)free(in);
-}
-
 static unsigned int _matrix_to_channelmask(int ch, char *matrix, char *premap, int **mout){
   unsigned int ret=0;
   char *p=matrix;
   int *perm=(*mout=malloc(ch*sizeof(*mout)));
   int i;
-  char **map = _tokenize_matrix(premap);
+  char **map;
 
+  if(!perm)
+    return 0;
+
+  map = _tokenize_matrix(premap);
+  if(!map)
+    return 0;
+
   for(i=0;i<ch;i++) perm[i] = -1;
   i=0;
 
@@ -811,6 +827,9 @@ static char *_channelmask_to_matrix(unsigned int mask,
   char buffer[257]={0};
   char **map = _tokenize_matrix(premap);
 
+  if(!map)
+    return NULL;
+
   while(map[m]){
     if(mask & (1<<m)){
       if(count)
@@ -850,6 +869,9 @@ static char *_matrix_intersect(char *matrix,char *prem
   int count=0;
   char **map = _tokenize_matrix(premap);
 
+  if(!map)
+    return NULL;
+
   while(1){
     char *h=p;
     int m=0;
@@ -1040,7 +1062,7 @@ static ao_device* _open_device(int driver_id, ao_sampl
                                                          device->output_matrix,
                                                          &device->input_map);
               int channels = _channelmask_bits(mask);
-              if(channels<0){
+              if(channels<=0){
                 aerror("Unable to map any channels from input matrix to output");
                 errno = AO_EBADFORMAT;
                 goto error;
@@ -1061,7 +1083,7 @@ static ao_device* _open_device(int driver_id, ao_sampl
                                                          device->output_matrix,
                                                          &device->input_map);
               int channels = _channelmask_bits(mask);
-              if(channels<0){
+              if(channels<=0){
                 aerror("Unable to map any channels from input matrix to output");
                 errno = AO_EBADFORMAT;
                 goto error;
@@ -1112,6 +1134,10 @@ static ao_device* _open_device(int driver_id, ao_sampl
             int count=0;
             device->inter_permute = calloc(device->output_channels,sizeof(int));
 
+            if (!device->inter_permute) {
+              errno = AO_EFAIL;
+              goto error;
+            }
             adebug("\n");
 
             while(count<device->output_channels){
@@ -1158,8 +1184,10 @@ static ao_device* _open_device(int driver_id, ao_sampl
                 for(i=0;i<device->output_channels;i++)
                   if(device->inter_permute[i]==j)break;
                 if(i==device->output_channels){
-                  adebug("input %d (%s)\t -> none\n",
-                         j,inch[j]);
+                  if(inch){
+                    adebug("input %d (%s)\t -> none\n",
+                           j,inch[j]);
+                  }
                   unflag=1;
                 }
               }
