Bug#976177: ghostscript: segfault in PDF->PNG conversion on big-endian triggered by ocrmypdf autopkgtest
Package: ghostscript
Version: 9.53.0~~rc1~dfsg-1
Severity: normal
Forwarded: https://bugs.ghostscript.com/show_bug.cgi?id=703164
The ocrmypdf autopkgtest triggers a segfault on s390x, in
tests/test_graft:test_no_graphless_graft
https://github.com/jbarlow83/OCRmyPDF/blob/master/tests/test_graft.py#L16
Input file (input.pdf) attached.
Command: gs -dQUIET -dSAFER -dBATCH -dNOPAUSE -sDEVICE=png16m -dFirstPage=3 -dLastPage=3 -r300.000000x300.000000 -o output.png -sstdout=%stderr -dAutoRotatePages=/None -f input.pdf
I can't reproduce the issue on upstream git, with their bundled lcms2mt,
but once I delete that, then I can. Reproduced on s390x and ppc64.
The patch landed in response to
https://bugs.ghostscript.com/show_bug.cgi?id=703164 fixes the bug, but breaks
support for non-bundled lcms2, so that lead to
https://bugs.ghostscript.com/show_bug.cgi?id=703208
Cherry-picking these two commits should fix the bug:
1. http://git.ghostscript.com/?p=ghostpdl.git;a=commitdiff;h=bd48c43be5f736393372dffbad627ed6fc486238
2. https://github.com/stefanor/ghostpdl/commit/c988b051115c1fac4e0d7f4c1bd807646d83b1ec
SR
From 47d803266a4c83264779840280d58d458fc2402d Mon Sep 17 00:00:00 2001
From: Michael Vrhel <michael.vrhel@artifex.com>
Date: Fri, 20 Nov 2020 11:45:50 -0800
Subject: [PATCH 1/2] Bug 703164: Endian issues with CMM
The interface code to the CMM was corrected to indicate when a
endian swap was needed on the data. This should only occur
in the case when we are dealing with transparency buffers
during the put image blending operation that may include
a color conversion. The final blend bakes the data as BE
so if we are on a LE machine, the CMM will need to know to
swap the bytes (assuming the pdf14 device is using 16bit buffers).
The code was rewritten to make it clear that this setting is no
BE vs LE but simply an endian swap. That was a source of confusion.
Revealed in this testing was the lack of some proper error
reporting during buffer conversions, which were fixed.
---
base/gdevp14.c | 57 ++++++++++++++++++++++++----------------
base/gscms.h | 2 +-
base/gsicc_cache.c | 7 +----
base/gsicc_lcms2mt.c | 33 ++++++++++-------------
base/gxblend.c | 5 ++--
base/gxblend.h | 2 +-
base/gxblend1.c | 5 +++-
base/gxi12bit.c | 8 ++++--
base/gxicolor.c | 8 ++++--
base/gxipixel.c | 8 ++++--
base/gxiscale.c | 18 ++++++++++---
devices/vector/gdevxps.c | 4 ++-
12 files changed, 94 insertions(+), 63 deletions(-)
diff --git a/base/gdevp14.c b/base/gdevp14.c
index 75e8fc71e..f161717e9 100644
--- a/base/gdevp14.c
+++ b/base/gdevp14.c
@@ -887,13 +887,14 @@ resolve_matte(pdf14_buf *maskbuf, byte *src_data, int src_planestride, int src_r
need to do the offset to our data in the buffer. Bug 700686: If we are in
a softmask that includes a matte entry, then we need to undo the matte
entry here at this time in the image's native color space not the parent
- color space. The big_endian term here is only set to true if the data
- has been baked as such during the put_image blending operation. */
+ color space. The endian_swap term here is only set to true if the data
+ has been baked as BE during the put_image blending operation and we are
+ on a LE machine. */
static forceinline pdf14_buf*
template_transform_color_buffer(gs_gstate *pgs, pdf14_ctx *ctx, gx_device *dev,
pdf14_buf *src_buf, byte *src_data, cmm_profile_t *src_profile,
cmm_profile_t *des_profile, int x0, int y0, int width, int height, bool *did_alloc,
- bool has_matte, bool deep, bool big_endian)
+ bool has_matte, bool deep, bool endian_swap)
{
gsicc_rendering_param_t rendering_params;
gsicc_link_t *icc_link;
@@ -913,6 +914,8 @@ template_transform_color_buffer(gs_gstate *pgs, pdf14_ctx *ctx, gx_device *dev,
pdf14_buf *output = src_buf;
pdf14_mask_t *mask_stack;
pdf14_buf *maskbuf;
+ int code;
+
*did_alloc = false;
/* Same profile */
@@ -970,10 +973,8 @@ template_transform_color_buffer(gs_gstate *pgs, pdf14_ctx *ctx, gx_device *dev,
gsicc_init_buffer(&des_buff_desc, des_profile->num_comps, 1<<deep, false,
false, true, des_planestride, des_rowstride, height, width);
- if (big_endian) {
- src_buff_desc.little_endian = false;
- des_buff_desc.little_endian = false;
- }
+ src_buff_desc.endian_swap = endian_swap;
+ des_buff_desc.endian_swap = endian_swap;
/* If we have a matte entry, undo the pre-blending now. Also set pdf14
context to ensure that this is not done again during the group
@@ -989,9 +990,11 @@ template_transform_color_buffer(gs_gstate *pgs, pdf14_ctx *ctx, gx_device *dev,
/* Transform the data. Since the pdf14 device should be using RGB, CMYK or
Gray buffers, this transform does not need to worry about the cmap procs
of the target device. */
- (icc_link->procs.map_buffer)(dev, icc_link, &src_buff_desc, &des_buff_desc,
+ code = (icc_link->procs.map_buffer)(dev, icc_link, &src_buff_desc, &des_buff_desc,
src_data, des_data);
gsicc_release_link(icc_link);
+ if (code < 0)
+ return NULL;
output->planestride = des_planestride;
output->rowstride = des_rowstride;
@@ -1045,28 +1048,28 @@ static pdf14_buf*
pdf14_transform_color_buffer_no_matte(gs_gstate *pgs, pdf14_ctx *ctx, gx_device *dev,
pdf14_buf *src_buf, byte *src_data, cmm_profile_t *src_profile,
cmm_profile_t *des_profile, int x0, int y0, int width, int height, bool *did_alloc,
- bool deep, bool big_endian)
+ bool deep, bool endian_swap)
{
if (deep)
return template_transform_color_buffer(pgs, ctx, dev, src_buf, src_data, src_profile,
- des_profile, x0, y0, width, height, did_alloc, false, true, big_endian);
+ des_profile, x0, y0, width, height, did_alloc, false, true, endian_swap);
else
return template_transform_color_buffer(pgs, ctx, dev, src_buf, src_data, src_profile,
- des_profile, x0, y0, width, height, did_alloc, false, false, big_endian);
+ des_profile, x0, y0, width, height, did_alloc, false, false, endian_swap);
}
static pdf14_buf*
pdf14_transform_color_buffer_with_matte(gs_gstate *pgs, pdf14_ctx *ctx, gx_device *dev,
pdf14_buf *src_buf, byte *src_data, cmm_profile_t *src_profile,
cmm_profile_t *des_profile, int x0, int y0, int width, int height, bool *did_alloc,
- bool deep, bool big_endian)
+ bool deep, bool endian_swap)
{
if (deep)
return template_transform_color_buffer(pgs, ctx, dev, src_buf, src_data, src_profile,
- des_profile, x0, y0, width, height, did_alloc, true, true, big_endian);
+ des_profile, x0, y0, width, height, did_alloc, true, true, endian_swap);
else
return template_transform_color_buffer(pgs, ctx, dev, src_buf, src_data, src_profile,
- des_profile, x0, y0, width, height, did_alloc, true, false, big_endian);
+ des_profile, x0, y0, width, height, did_alloc, true, false, endian_swap);
}
/**
@@ -1908,6 +1911,7 @@ pdf14_pop_transparency_mask(pdf14_ctx *ctx, gs_gstate *pgs, gx_device *dev)
gsicc_link_t *icc_link;
gsicc_rendering_param_t render_cond;
cmm_dev_profile_t *dev_profile;
+ int code = 0;
dev_proc(dev, get_profile)(dev, &dev_profile);
gsicc_extract_profile(GS_UNKNOWN_TAG, dev_profile, &src_profile,
@@ -2058,7 +2062,7 @@ pdf14_pop_transparency_mask(pdf14_ctx *ctx, gs_gstate *pgs, gx_device *dev)
rendering_params.cmm = gsCMM_DEFAULT;
icc_link = gsicc_get_link_profile(pgs, dev, des_profile,
src_profile, &rendering_params, pgs->memory, false);
- smask_icc(dev, tos->rect.q.y - tos->rect.p.y,
+ code = smask_icc(dev, tos->rect.q.y - tos->rect.p.y,
tos->rect.q.x - tos->rect.p.x, tos->n_chan,
tos->rowstride, tos->planestride,
tos->data, new_data_buf, icc_link, tos->deep);
@@ -2089,7 +2093,7 @@ pdf14_pop_transparency_mask(pdf14_ctx *ctx, gs_gstate *pgs, gx_device *dev)
return gs_note_error(gs_error_VMerror);
ctx->mask_stack->rc_mask->mask_buf = tos;
}
- return 0;
+ return code;
}
static pdf14_mask_t *
@@ -2407,7 +2411,7 @@ pdf14_put_image_color_convert(const pdf14_device* dev, gs_gstate* pgs, cmm_profi
cmm_profile_t* des_profile;
gsicc_rendering_param_t render_cond;
bool did_alloc;
- bool big_endian;
+ bool endian_swap;
gsicc_extract_profile(GS_UNKNOWN_TAG, dev_target_profile, &des_profile,
&render_cond);
@@ -2419,17 +2423,26 @@ pdf14_put_image_color_convert(const pdf14_device* dev, gs_gstate* pgs, cmm_profi
global_index++;
#endif
- /* If we are doing a 16 bit buffer it will be big endian if we have already done the
- blend, otherwise it will be native endian */
+ /* If we are doing a 16 bit buffer it will be big endian if we have already done the
+ blend, otherwise it will be native endian. GS expects its 16bit buffers to be BE
+ but for sanity pdf14 device maintains 16bit buffers in native format. The CMM
+ will need to know if it is dealing with native or BE data. */
if (was_blended && (*buf)->deep) {
- big_endian = true;
+ /* Data is in BE. If we are in a LE machine, CMM will need to swap for
+ color conversion */
+#if ARCH_IS_BIG_ENDIAN
+ endian_swap = false;
+#else
+ endian_swap = true;
+#endif
} else {
- big_endian = false;
+ /* Data is in native format. No swap needed for CMM */
+ endian_swap = false;
}
cm_result = pdf14_transform_color_buffer_no_matte(pgs, dev->ctx, (gx_device*) dev, *buf,
*buf_ptr, src_profile, des_profile, x, y, width,
- height, &did_alloc, (*buf)->deep, big_endian);
+ height, &did_alloc, (*buf)->deep, endian_swap);
if (cm_result == NULL)
return_error(gs_error_VMerror);
diff --git a/base/gscms.h b/base/gscms.h
index dfface693..f22c60a30 100644
--- a/base/gscms.h
+++ b/base/gscms.h
@@ -86,7 +86,7 @@ typedef struct gsicc_bufferdesc_s {
unsigned char bytes_per_chan;
bool has_alpha;
bool alpha_first;
- bool little_endian;
+ bool endian_swap;
bool is_planar;
int plane_stride;
int row_stride;
diff --git a/base/gsicc_cache.c b/base/gsicc_cache.c
index df51e88b8..bf8e6a22d 100644
--- a/base/gsicc_cache.c
+++ b/base/gsicc_cache.c
@@ -1788,12 +1788,7 @@ gsicc_init_buffer(gsicc_bufferdesc_t *buffer_desc, unsigned char num_chan, unsig
buffer_desc->row_stride = row_stride;
buffer_desc->num_rows = num_rows;
buffer_desc->pixels_per_row = pixels_per_row;
-
-#if ARCH_IS_BIG_ENDIAN
- buffer_desc->little_endian = false;
-#else
- buffer_desc->little_endian = true;
-#endif
+ buffer_desc->endian_swap = false;
}
/* Return the proper component numbers based upon the profiles of the device.
diff --git a/base/gsicc_lcms2mt.c b/base/gsicc_lcms2mt.c
index dbb9c4b2a..dffb76d67 100644
--- a/base/gsicc_lcms2mt.c
+++ b/base/gsicc_lcms2mt.c
@@ -40,10 +40,10 @@
#define LCMS_BYTES_MASK T_BYTES(-1) /* leaves only mask for the BYTES (currently 7) */
#define LCMS_ENDIAN16_MASK T_ENDIAN16(-1) /* similarly, for ENDIAN16 bit */
-#define gsicc_link_flags(hasalpha, planarIN, planarOUT, bigendianIN, bigendianOUT, bytesIN, bytesOUT) \
+#define gsicc_link_flags(hasalpha, planarIN, planarOUT, endianswapIN, endianswapOUT, bytesIN, bytesOUT) \
((hasalpha != 0) << 2 | \
(planarIN != 0) << 5 | (planarOUT != 0) << 4 | \
- (bigendianIN != 0) << 3 | (bigendianOUT != 0) << 2 | \
+ (endianswapIN != 0) << 3 | (endianswapOUT != 0) << 2 | \
(bytesIN == 1) << 1 | (bytesOUT == 1))
typedef struct gsicc_lcms2mt_link_list_s {
@@ -338,7 +338,7 @@ gscms_transform_color_buffer(gx_device *dev, gsicc_link_t *icclink,
gsicc_lcms2mt_link_list_t *link_handle = (gsicc_lcms2mt_link_list_t *)(icclink->link_handle);
cmsHTRANSFORM hTransform = (cmsHTRANSFORM)link_handle->hTransform;
cmsUInt32Number dwInputFormat, dwOutputFormat, num_src_lcms, num_des_lcms;
- int hasalpha, planarIN, planarOUT, numbytesIN, numbytesOUT, big_endianIN, big_endianOUT;
+ int hasalpha, planarIN, planarOUT, numbytesIN, numbytesOUT, swap_endianIN, swap_endianOUT;
int needed_flags = 0;
unsigned char *inputpos, *outputpos;
cmsContext ctx = gs_lib_ctx_get_cms_context(icclink->memory);
@@ -349,7 +349,7 @@ gscms_transform_color_buffer(gx_device *dev, gsicc_link_t *icclink,
/* Although little CMS does make assumptions about data types in its
transformations we can change it after the fact by cloning from any
other transform. We always create [0] which is no_alpha, chunky IN/OUT,
- little_endian IN/OUT, 2-bytes_per_component IN/OUT. */
+ no endian swap IN/OUT, 2-bytes_per_component IN/OUT. */
/* Set us to the proper output type */
/* Note, we could speed this up by passing back the encoded data type
to the caller so that we could avoid having to go through this
@@ -368,8 +368,8 @@ gscms_transform_color_buffer(gx_device *dev, gsicc_link_t *icclink,
return_error(gs_error_rangecheck); /* TODO: we don't support float */
/* endian */
- big_endianIN = !input_buff_desc->little_endian;
- big_endianOUT = !output_buff_desc->little_endian;
+ swap_endianIN = input_buff_desc->endian_swap;
+ swap_endianOUT = output_buff_desc->endian_swap;
/* alpha, which is passed through unmolested */
/* TODO: Right now we always must have alpha last */
@@ -377,7 +377,7 @@ gscms_transform_color_buffer(gx_device *dev, gsicc_link_t *icclink,
hasalpha = input_buff_desc->has_alpha;
needed_flags = gsicc_link_flags(hasalpha, planarIN, planarOUT,
- big_endianIN, big_endianOUT,
+ swap_endianIN, swap_endianOUT,
numbytesIN, numbytesOUT);
while (link_handle->flags != needed_flags) {
if (link_handle->next == NULL) {
@@ -418,8 +418,8 @@ gscms_transform_color_buffer(gx_device *dev, gsicc_link_t *icclink,
dwOutputFormat = dwOutputFormat | EXTRA_SH(hasalpha);
dwInputFormat = dwInputFormat | PLANAR_SH(planarIN);
dwOutputFormat = dwOutputFormat | PLANAR_SH(planarOUT);
- dwInputFormat = dwInputFormat | ENDIAN16_SH(big_endianIN);
- dwOutputFormat = dwOutputFormat | ENDIAN16_SH(big_endianOUT);
+ dwInputFormat = dwInputFormat | ENDIAN16_SH(swap_endianIN);
+ dwOutputFormat = dwOutputFormat | ENDIAN16_SH(swap_endianOUT);
dwInputFormat = dwInputFormat | BYTES_SH(numbytesIN);
dwOutputFormat = dwOutputFormat | BYTES_SH(numbytesOUT);
@@ -608,9 +608,7 @@ gscms_get_link(gcmmhprofile_t lcms_srchandle, gcmmhprofile_t lcms_deshandle,
when we use the transformation. */
src_data_type = (COLORSPACE_SH(lcms_src_color_space)|
CHANNELS_SH(src_nChannels)|BYTES_SH(2));
-#if 0
- src_data_type = src_data_type | ENDIAN16_SH(1);
-#endif
+
if (lcms_deshandle != NULL) {
des_color_space = cmsGetColorSpace(ctx, lcms_deshandle);
} else {
@@ -623,10 +621,7 @@ gscms_get_link(gcmmhprofile_t lcms_srchandle, gcmmhprofile_t lcms_deshandle,
des_nChannels = cmsChannelsOf(ctx, des_color_space);
des_data_type = (COLORSPACE_SH(lcms_des_color_space)|
CHANNELS_SH(des_nChannels)|BYTES_SH(2));
- /* endian */
-#if 0
- des_data_type = des_data_type | ENDIAN16_SH(1);
-#endif
+
/* Set up the flags */
flag = gscms_get_accuracy(memory);
if (rendering_params->black_point_comp == gsBLACKPTCOMP_ON
@@ -679,7 +674,7 @@ gscms_get_link(gcmmhprofile_t lcms_srchandle, gcmmhprofile_t lcms_deshandle,
return NULL;
}
link_handle->next = NULL;
- link_handle->flags = gsicc_link_flags(0, 0, 0, 0, 0, /* no alpha, not planar, little-endian */
+ link_handle->flags = gsicc_link_flags(0, 0, 0, 0, 0, /* no alpha, not planar, no endian swap */
sizeof(gx_color_value), sizeof(gx_color_value));
return link_handle;
/* cmsFLAGS_HIGHRESPRECALC) cmsFLAGS_NOTPRECALC cmsFLAGS_LOWRESPRECALC*/
@@ -714,7 +709,7 @@ gscms_get_link_proof_devlink(gcmmhprofile_t lcms_srchandle,
if (link_handle == NULL)
return NULL;
link_handle->next = NULL;
- link_handle->flags = gsicc_link_flags(0, 0, 0, 0, 0, /* no alpha, not planar, little-endian */
+ link_handle->flags = gsicc_link_flags(0, 0, 0, 0, 0, /* no alpha, not planar, no endian swap */
sizeof(gx_color_value), sizeof(gx_color_value));
/* Check if the rendering intent is something other than relative colorimetric
and if we have a proofing profile. In this case we need to create the
@@ -1028,7 +1023,7 @@ gscms_get_name2device_link(gsicc_link_t *icclink,
cmsDeleteTransform(ctx, hTransformNew);
return; /* bail */
}
- link_handle->flags = gsicc_link_flags(0, 0, 0, 0, 0, /* no alpha, not planar, little-endian */
+ link_handle->flags = gsicc_link_flags(0, 0, 0, 0, 0, /* no alpha, not planar, no endian swap */
sizeof(gx_color_value), sizeof(gx_color_value));
link_handle->hTransform = hTransformNew;
link_handle->next = NULL;
diff --git a/base/gxblend.c b/base/gxblend.c
index 410594836..c66650ac8 100644
--- a/base/gxblend.c
+++ b/base/gxblend.c
@@ -402,7 +402,8 @@ void smask_copy(int num_rows, int num_cols, int row_stride,
}
}
-void smask_icc(gx_device *dev, int num_rows, int num_cols, int n_chan,
+int
+smask_icc(gx_device *dev, int num_rows, int num_cols, int n_chan,
int row_stride, int plane_stride, byte *gs_restrict src, const byte *gs_restrict dst,
gsicc_link_t *icclink, bool deep)
{
@@ -427,7 +428,7 @@ void smask_icc(gx_device *dev, int num_rows, int num_cols, int n_chan,
false, false, true, plane_stride,
row_stride, num_rows, num_cols);
/* Transform the data */
- (icclink->procs.map_buffer)(dev, icclink, &input_buff_desc, &output_buff_desc,
+ return (icclink->procs.map_buffer)(dev, icclink, &input_buff_desc, &output_buff_desc,
(void*) src, (void*) dst);
}
diff --git a/base/gxblend.h b/base/gxblend.h
index 0ac938776..42874adbc 100644
--- a/base/gxblend.h
+++ b/base/gxblend.h
@@ -100,7 +100,7 @@ void smask_blend(byte *gs_restrict src, int width, int height, int rowstride,
void smask_copy(int num_rows, int num_cols, int row_stride,
byte *gs_restrict src, const byte *gs_restrict des);
-void smask_icc(gx_device *dev, int num_rows, int num_cols, int n_chan,
+int smask_icc(gx_device *dev, int num_rows, int num_cols, int n_chan,
int row_stride, int plane_stride, byte *gs_restrict src, const byte *gs_restrict des,
gsicc_link_t *icclink, bool deep);
/* For spot colors, blend modes must be white preserving and separable */
diff --git a/base/gxblend1.c b/base/gxblend1.c
index 6d42a0941..cb14f5240 100644
--- a/base/gxblend1.c
+++ b/base/gxblend1.c
@@ -226,6 +226,7 @@ pdf14_preserve_backdrop_cm(pdf14_buf *buf, cmm_profile_t *group_profile,
int y0 = max(buf->rect.p.y, tos->rect.p.y);
int y1 = min(buf->rect.q.y, tos->rect.q.y);
bool deep = buf->deep;
+ int code;
if (x0 < x1 && y0 < y1) {
int width = x1 - x0;
@@ -283,9 +284,11 @@ pdf14_preserve_backdrop_cm(pdf14_buf *buf, cmm_profile_t *group_profile,
false, true, buf->planestride, buf->rowstride, height,
width);
/* Transform the data. */
- (icc_link->procs.map_buffer)(dev, icc_link, &input_buff_desc,
+ code = (icc_link->procs.map_buffer)(dev, icc_link, &input_buff_desc,
&output_buff_desc, tos_plane, buf_plane);
gsicc_release_link(icc_link);
+ if (code < 0)
+ return gs_throw(gs_error_unknownerror, "ICC transform failed. Trans backdrop");
}
/* Copy the alpha data */
buf_plane += buf->planestride * (buf->n_chan - 1);
diff --git a/base/gxi12bit.c b/base/gxi12bit.c
index fdf56fb19..f46b2d261 100644
--- a/base/gxi12bit.c
+++ b/base/gxi12bit.c
@@ -666,19 +666,23 @@ image_render_icc16(gx_image_enum * penum, const byte * buffer, int data_x,
(const unsigned short*) (psrc_decode+w),
get_cie_range(penum->pcs));
}
- (penum->icc_link->procs.map_buffer)(dev, penum->icc_link,
+ code = (penum->icc_link->procs.map_buffer)(dev, penum->icc_link,
&input_buff_desc,
&output_buff_desc,
(void*) psrc_decode,
(void*) psrc_cm);
gs_free_object(pgs->memory, (byte *)psrc_decode, "image_render_color_icc");
+ if (code < 0)
+ return code;
} else {
/* CM only. No decode */
- (penum->icc_link->procs.map_buffer)(dev, penum->icc_link,
+ code = (penum->icc_link->procs.map_buffer)(dev, penum->icc_link,
&input_buff_desc,
&output_buff_desc,
(void*) psrc,
(void*) psrc_cm);
+ if (code < 0)
+ return code;
}
}
}
diff --git a/base/gxicolor.c b/base/gxicolor.c
index 58ead88af..432f6e853 100644
--- a/base/gxicolor.c
+++ b/base/gxicolor.c
@@ -566,19 +566,23 @@ image_color_icc_prep(gx_image_enum *penum_orig, const byte *psrc, uint w,
decode_row_cie(penum, psrc, spp, psrc_decode,
psrc_decode+w, get_cie_range(penum->pcs));
}
- (penum->icc_link->procs.map_buffer)(dev, penum->icc_link,
+ code = (penum->icc_link->procs.map_buffer)(dev, penum->icc_link,
&input_buff_desc,
&output_buff_desc,
(void*) psrc_decode,
(void*) *psrc_cm);
gs_free_object(pgs->memory, psrc_decode, "image_color_icc_prep");
+ if (code < 0)
+ return code;
} else {
/* CM only. No decode */
- (penum->icc_link->procs.map_buffer)(dev, penum->icc_link,
+ code = (penum->icc_link->procs.map_buffer)(dev, penum->icc_link,
&input_buff_desc,
&output_buff_desc,
(void*) psrc,
(void*) *psrc_cm);
+ if (code < 0)
+ return code;
}
}
}
diff --git a/base/gxipixel.c b/base/gxipixel.c
index c60575770..927dea068 100644
--- a/base/gxipixel.c
+++ b/base/gxipixel.c
@@ -1133,6 +1133,7 @@ image_init_color_cache(gx_image_enum * penum, int bps, int spp)
gsicc_bufferdesc_t input_buff_desc;
gsicc_bufferdesc_t output_buff_desc;
gx_color_value conc[GX_DEVICE_COLOR_MAX_COMPONENTS];
+ int code;
if (penum->icc_link == NULL) {
return gs_rethrow(-1, "ICC Link not created during image render color");
@@ -1265,10 +1266,13 @@ image_init_color_cache(gx_image_enum * penum, int bps, int spp)
gsicc_init_buffer(&output_buff_desc, num_des_comp, 1, false, false, false,
0, num_entries * num_des_comp,
1, num_entries);
- (penum->icc_link->procs.map_buffer)(penum->dev, penum->icc_link,
- &input_buff_desc, &output_buff_desc,
+ code = (penum->icc_link->procs.map_buffer)(penum->dev, penum->icc_link,
+ &input_buff_desc, &output_buff_desc,
(void*) temp_buffer,
(void*) penum->color_cache->device_contone);
+ if (code < 0)
+ return gs_rethrow(code, "Failure to map color buffer");
+
/* Check if we need to apply any transfer functions. If so then do it now */
if (has_transfer) {
for (k = 0; k < num_entries; k++) {
diff --git a/base/gxiscale.c b/base/gxiscale.c
index 834274319..16bcd5840 100644
--- a/base/gxiscale.c
+++ b/base/gxiscale.c
@@ -2171,9 +2171,12 @@ image_render_interpolate_icc(gx_image_enum * penum, const byte * buffer,
1, width_in);
/* Do the transformation */
psrc = (byte*) (stream_r.ptr + 1);
- (penum->icc_link->procs.map_buffer)(dev, penum->icc_link, &input_buff_desc,
+ code = (penum->icc_link->procs.map_buffer)(dev, penum->icc_link, &input_buff_desc,
&output_buff_desc, (void*) psrc,
(void*) p_cm_buff);
+ if (code < 0)
+ return code;
+
/* Re-set the reading stream to use the cm data */
stream_r.ptr = p_cm_buff - 1;
stream_r.limit = stream_r.ptr + num_bytes_decode * width_in * spp_cm;
@@ -2232,11 +2235,13 @@ image_render_interpolate_icc(gx_image_enum * penum, const byte * buffer,
pinterp += (pss->params.LeftMarginOut / abs_interp_limit) * spp_decode;
p_cm_interp = (unsigned short *) p_cm_buff;
p_cm_interp += (pss->params.LeftMarginOut / abs_interp_limit) * spp_cm;
- (penum->icc_link->procs.map_buffer)(dev, penum->icc_link,
+ code = (penum->icc_link->procs.map_buffer)(dev, penum->icc_link,
&input_buff_desc,
&output_buff_desc,
(void*) pinterp,
(void*) p_cm_interp);
+ if (code < 0)
+ return code;
}
code = irii_core(penum, xo, xe, spp_cm, p_cm_interp, dev, abs_interp_limit, bpp, raster, yo, dy, lop);
if (code < 0)
@@ -2660,9 +2665,12 @@ image_render_interpolate_landscape_icc(gx_image_enum * penum,
1, width_in);
/* Do the transformation */
psrc = (byte*) (stream_r.ptr + 1);
- (penum->icc_link->procs.map_buffer)(dev, penum->icc_link, &input_buff_desc,
+ code = (penum->icc_link->procs.map_buffer)(dev, penum->icc_link, &input_buff_desc,
&output_buff_desc, (void*) psrc,
(void*) p_cm_buff);
+ if (code < 0)
+ return code;
+
/* Re-set the reading stream to use the cm data */
stream_r.ptr = p_cm_buff - 1;
stream_r.limit = stream_r.ptr + num_bytes_decode * width_in * spp_cm;
@@ -2733,11 +2741,13 @@ image_render_interpolate_landscape_icc(gx_image_enum * penum,
pinterp += (pss->params.LeftMarginOut / abs_interp_limit) * spp_decode;
p_cm_interp = (unsigned short *) p_cm_buff;
p_cm_interp += (pss->params.LeftMarginOut / abs_interp_limit) * spp_cm;
- (penum->icc_link->procs.map_buffer)(dev, penum->icc_link,
+ code = (penum->icc_link->procs.map_buffer)(dev, penum->icc_link,
&input_buff_desc,
&output_buff_desc,
(void*) pinterp,
(void*) p_cm_interp);
+ if (code < 0)
+ return code;
}
p_cm_interp += (pss->params.LeftMarginOut / abs_interp_limit) * spp_cm;
for (x = xo; x < xe;) {
diff --git a/devices/vector/gdevxps.c b/devices/vector/gdevxps.c
index 8eb26196d..7a895cd1d 100644
--- a/devices/vector/gdevxps.c
+++ b/devices/vector/gdevxps.c
@@ -2173,9 +2173,11 @@ const gx_image_plane_t *planes, int height, int *rows_used)
gsicc_init_buffer(&output_buff_desc, 3, bytes_comp,
false, false, false, 0, width * bytes_comp * 3,
1, width);
- (pie->icc_link->procs.map_buffer)(pie->dev, pie->icc_link,
+ code = (pie->icc_link->procs.map_buffer)(pie->dev, pie->icc_link,
&input_buff_desc, &output_buff_desc, (void*)buffer,
(void*)pie->buffer);
+ if (code < 0)
+ return code;
outbuffer = pie->buffer;
}
code = TIFFWriteScanline(pie->tif, outbuffer, pie->y, 0);
--
2.20.1
From 6c8bdb76e822bdfca939b4e10f81ec86ea9ae88f Mon Sep 17 00:00:00 2001
From: Stefano Rivera <stefano@rivera.za.net>
Date: Mon, 30 Nov 2020 16:04:01 -0800
Subject: [PATCH 2/2] Update lcms2 (non-mt) code for buff_desc->endian_swap
Forgotten in bd48c43be5f736393372dffbad627ed6fc486238
---
base/gsicc_lcms2.c | 17 +++++------------
1 file changed, 5 insertions(+), 12 deletions(-)
diff --git a/base/gsicc_lcms2.c b/base/gsicc_lcms2.c
index 43a7a93a8..f09fbc6cc 100644
--- a/base/gsicc_lcms2.c
+++ b/base/gsicc_lcms2.c
@@ -308,7 +308,7 @@ gscms_transform_color_buffer(gx_device *dev, gsicc_link_t *icclink,
{
cmsHTRANSFORM hTransform = (cmsHTRANSFORM)icclink->link_handle;
cmsUInt32Number dwInputFormat, dwOutputFormat, num_src_lcms, num_des_lcms;
- int planar,numbytes, big_endian, hasalpha, k;
+ int planar,numbytes, swap_endian, hasalpha, k;
unsigned char *inputpos, *outputpos;
#if DUMP_CMS_BUFFER
gp_file *fid_in, *fid_out;
@@ -340,10 +340,10 @@ gscms_transform_color_buffer(gx_device *dev, gsicc_link_t *icclink,
dwOutputFormat = dwOutputFormat | BYTES_SH(numbytes);
/* endian */
- big_endian = !input_buff_desc->little_endian;
- dwInputFormat = dwInputFormat | ENDIAN16_SH(big_endian);
- big_endian = !output_buff_desc->little_endian;
- dwOutputFormat = dwOutputFormat | ENDIAN16_SH(big_endian);
+ swap_endian = input_buff_desc->endian_swap;
+ dwInputFormat = dwInputFormat | ENDIAN16_SH(swap_endian);
+ swap_endian = output_buff_desc->endian_swap;
+ dwOutputFormat = dwOutputFormat | ENDIAN16_SH(swap_endian);
/* number of channels. This should not really be changing! */
num_src_lcms = T_CHANNELS(cmsGetTransformInputFormat(hTransform));
@@ -524,9 +524,6 @@ gscms_get_link(gcmmhprofile_t lcms_srchandle,
when we use the transformation. */
src_data_type = (COLORSPACE_SH(lcms_src_color_space)|
CHANNELS_SH(src_nChannels)|BYTES_SH(2));
-#if 0
- src_data_type = src_data_type | ENDIAN16_SH(1);
-#endif
if (lcms_deshandle != NULL) {
des_color_space = cmsGetColorSpace(lcms_deshandle);
} else {
@@ -538,10 +535,6 @@ gscms_get_link(gcmmhprofile_t lcms_srchandle,
des_nChannels = cmsChannelsOf(des_color_space);
des_data_type = (COLORSPACE_SH(lcms_des_color_space)|
CHANNELS_SH(des_nChannels)|BYTES_SH(2));
- /* endian */
-#if 0
- des_data_type = des_data_type | ENDIAN16_SH(1);
-#endif
/* Set up the flags */
flag = cmsFLAGS_HIGHRESPRECALC;
if (rendering_params->black_point_comp == gsBLACKPTCOMP_ON
--
2.20.1
Reply to: