Module refinery.units.formats.jpeg

Classes

class JpegMarker (*args, **kwds)

The markers that the JPEG standard defines.

Expand source code Browse git

class JpegMarker(int, enum.Enum):
    """
    The markers that the JPEG standard defines.
    """
    StartOfImage = 0xD8
    EndOfImage = 0xD9
    StartOfScan = 0xDA
    DCTBaseline = 0xC0
    DCTExtendedSequential = 0xC1
    DCTProgressive = 0xC2
    Lossless = 0xC3
    DifferentialDCTSequential = 0xC5
    DifferentialDCTProgressive = 0xC6
    DifferentialLossless = 0xC7
    Extension = 0xC8
    ArithmeticDCTExtendedSequential = 0xC9
    ArithmeticDCTProgressive = 0xCA
    ArithmeticLossless = 0xCB
    ArithmeticDifferentialDCTSequential = 0xCD
    ArithmeticDifferentialDCTProgressive = 0xCE
    ArithmeticDifferentialLossless = 0xCF
    DefineQuantizationTable = 0xDB
    DefineHuffmanTable = 0xC4
    DefineArithmeticConditioning = 0xCC
    DefineRestartInterval = 0xDD
    DefineNumberOfLines = 0xDC
    DefineHierarchicalProgression = 0xDE
    ExpandReferenceComponents = 0xDF
    Comment = 0xFE
    Temporary = 0x01

    RST0 = 0xD0
    RST1 = 0xD1
    RST2 = 0xD2
    RST3 = 0xD3
    RST4 = 0xD4
    RST5 = 0xD5
    RST6 = 0xD6
    RST7 = 0xD7

    App00 = 0xE0
    App01 = 0xE1
    App02 = 0xE2
    App03 = 0xE3
    App04 = 0xE4
    App05 = 0xE5
    App06 = 0xE6
    App07 = 0xE7
    App08 = 0xE8
    App09 = 0xE9
    App10 = 0xEA
    App11 = 0xEB
    App12 = 0xEC
    App13 = 0xED
    App14 = 0xEE
    App15 = 0xEF

    Ext00 = 0xF0
    Ext01 = 0xF1
    Ext02 = 0xF2
    Ext03 = 0xF3
    Ext04 = 0xF4
    Ext05 = 0xF5
    Ext06 = 0xF6
    Ext07 = 0xF7
    Ext08 = 0xF8
    Ext09 = 0xF9
    Ext10 = 0xFA
    Ext11 = 0xFB
    Ext12 = 0xFC
    Ext13 = 0xFD

    SOI = StartOfImage
    EOI = EndOfImage
    SOS = StartOfScan
    DQT = DefineQuantizationTable
    DHT = DefineHuffmanTable
    DAC = DefineArithmeticConditioning
    DRI = DefineRestartInterval
    DNL = DefineNumberOfLines
    DHP = DefineHierarchicalProgression
    EXP = ExpandReferenceComponents
    COM = Comment
    TEM = Temporary
    SOF0 = DCTBaseline
    SOF1 = DCTExtendedSequential
    SOF2 = DCTProgressive
    SOF3 = Lossless

    @property
    def is_standalone(self) -> bool:
        """
        Whether the marker stands alone, i.e. it is not followed by a segment.
        """
        return self == JpegMarker.TEM or self in range(0xD0, 0xDA)

    @property
    def is_frame(self) -> bool:
        """
        Whether the marker starts a frame header, which gives the dimensions of the image.
        """
        return self in range(0xC0, 0xD0) and self not in (
            JpegMarker.DHT,
            JpegMarker.Extension,
            JpegMarker.DAC,
        )

Ancestors

• builtins.int
• enum.Enum

Class variables

var StartOfImage

The type of the None singleton.

var EndOfImage

The type of the None singleton.

var StartOfScan

The type of the None singleton.

var DCTBaseline

The type of the None singleton.

var DCTExtendedSequential

The type of the None singleton.

var DCTProgressive

The type of the None singleton.

var Lossless

The type of the None singleton.

var DifferentialDCTSequential

The type of the None singleton.

var DifferentialDCTProgressive

The type of the None singleton.

var DifferentialLossless

The type of the None singleton.

var Extension

The type of the None singleton.

var ArithmeticDCTExtendedSequential

The type of the None singleton.

var ArithmeticDCTProgressive

The type of the None singleton.

var ArithmeticLossless

The type of the None singleton.

var ArithmeticDifferentialDCTSequential

The type of the None singleton.

var ArithmeticDifferentialDCTProgressive

The type of the None singleton.

var ArithmeticDifferentialLossless

The type of the None singleton.

var DefineQuantizationTable

The type of the None singleton.

var DefineHuffmanTable

The type of the None singleton.

var DefineArithmeticConditioning

The type of the None singleton.

var DefineRestartInterval

The type of the None singleton.

var DefineNumberOfLines

The type of the None singleton.

var DefineHierarchicalProgression

The type of the None singleton.

var ExpandReferenceComponents

The type of the None singleton.

var Comment

The type of the None singleton.

var Temporary

The type of the None singleton.

var RST0

The type of the None singleton.

var RST1

The type of the None singleton.

var RST2

The type of the None singleton.

var RST3

The type of the None singleton.

var RST4

The type of the None singleton.

var RST5

The type of the None singleton.

var RST6

The type of the None singleton.

var RST7

The type of the None singleton.

var App00

The type of the None singleton.

var App01

The type of the None singleton.

var App02

The type of the None singleton.

var App03

The type of the None singleton.

var App04

The type of the None singleton.

var App05

The type of the None singleton.

var App06

The type of the None singleton.

var App07

The type of the None singleton.

var App08

The type of the None singleton.

var App09

The type of the None singleton.

var App10

The type of the None singleton.

var App11

The type of the None singleton.

var App12

The type of the None singleton.

var App13

The type of the None singleton.

var App14

The type of the None singleton.

var App15

The type of the None singleton.

var Ext00

The type of the None singleton.

var Ext01

The type of the None singleton.

var Ext02

The type of the None singleton.

var Ext03

The type of the None singleton.

var Ext04

The type of the None singleton.

var Ext05

The type of the None singleton.

var Ext06

The type of the None singleton.

var Ext07

The type of the None singleton.

var Ext08

The type of the None singleton.

var Ext09

The type of the None singleton.

var Ext10

The type of the None singleton.

var Ext11

The type of the None singleton.

var Ext12

The type of the None singleton.

var Ext13

The type of the None singleton.

var SOI

The type of the None singleton.

var EOI

The type of the None singleton.

var SOS

The type of the None singleton.

var DQT

The type of the None singleton.

var DHT

The type of the None singleton.

var DAC

The type of the None singleton.

var DRI

The type of the None singleton.

var DNL

The type of the None singleton.

var DHP

The type of the None singleton.

var EXP

The type of the None singleton.

var COM

The type of the None singleton.

var TEM

The type of the None singleton.

var SOF0

The type of the None singleton.

var SOF1

The type of the None singleton.

var SOF2

The type of the None singleton.

var SOF3

The type of the None singleton.

Instance variables

var is_standalone

Whether the marker stands alone, i.e. it is not followed by a segment.

Expand source code Browse git

@property
def is_standalone(self) -> bool:
    """
    Whether the marker stands alone, i.e. it is not followed by a segment.
    """
    return self == JpegMarker.TEM or self in range(0xD0, 0xDA)

var is_frame

Whether the marker starts a frame header, which gives the dimensions of the image.

Expand source code Browse git

@property
def is_frame(self) -> bool:
    """
    Whether the marker starts a frame header, which gives the dimensions of the image.
    """
    return self in range(0xC0, 0xD0) and self not in (
        JpegMarker.DHT,
        JpegMarker.Extension,
        JpegMarker.DAC,
    )

class JpegSOFComponent (reader)

A class to parse structured data. A Struct class can be instantiated as follows:

foo = Struct(data, bar=29)

The initialization routine of the structure will be called with a single argument reader. If the object data is already a StructReader, then it will be passed as reader. Otherwise, the argument will be wrapped in a StructReader. Additional arguments to the struct are passed through.

Expand source code Browse git

class JpegSOFComponent(Struct):
    def __init__(self, reader: StructReader):
        self.id = reader.u8()
        hv = reader.u8()
        self.hs = hv >> 4
        self.vs = hv & 15
        self.qt = reader.u8()

Ancestors

• Struct
• typing.Generic
• collections.abc.Buffer

Static methods

def Parse(reader, *args, **kwargs)

class JpegSOF (reader)

A class to parse structured data. A Struct class can be instantiated as follows:

foo = Struct(data, bar=29)

The initialization routine of the structure will be called with a single argument reader. If the object data is already a StructReader, then it will be passed as reader. Otherwise, the argument will be wrapped in a StructReader. Additional arguments to the struct are passed through.

Expand source code Browse git

class JpegSOF(Struct):
    def __init__(self, reader: StructReader):
        reader.bigendian = True
        self.precision = reader.u8()
        self.height = reader.u16()
        self.width = reader.u16()
        self.components = [JpegSOFComponent(reader) for _ in range(reader.u8())]

Ancestors

• Struct
• typing.Generic
• collections.abc.Buffer

Static methods

def Parse(reader, *args, **kwargs)

class JpegStream (reader)

A marker with the fill bytes in front of it and the segment that follows it. The segment of a scan is followed by its entropy-coded data, which runs across restart markers up to the next marker of any other kind or to the end of the data. A segment that the end of the data cuts holds the bytes that are there.

Expand source code Browse git

class JpegStream(Struct):
    """
    A marker with the fill bytes in front of it and the segment that follows it. The segment of a
    scan is followed by its entropy-coded data, which runs across restart markers up to the next
    marker of any other kind or to the end of the data. A segment that the end of the data cuts
    holds the bytes that are there.
    """
    def __init__(self, reader: StructReader):
        self.offset = reader.tell()
        if (h := reader.u8()) != 0xFF:
            raise ValueError(F'Invalid magic byte {h:#04x} at start of stream.')
        while (marker := reader.u8()) == 0xFF:
            pass
        self.type = t = JpegMarker(marker)
        if t.is_standalone:
            self.size = 0
            self.data = b''
            self.scan = b''
        else:
            self.size = reader.u16() - 2
            if self.size < 0:
                raise ValueError(F'Invalid size {self.size}.')
            self.data = reader.read(self.size)
            if t == JpegMarker.StartOfScan:
                end = _END_OF_SCAN.search(reader.peek())
                self.scan = reader.read(None if end is None else end.start())
            else:
                self.scan = b''

    @property
    def truncated(self) -> bool:
        """
        Whether the end of the data cuts the segment.
        """
        return len(self.data) < self.size

Ancestors

• Struct
• typing.Generic
• collections.abc.Buffer

Static methods

def Parse(reader, *args, **kwargs)

Instance variables

var truncated

Whether the end of the data cuts the segment.

Expand source code Browse git

@property
def truncated(self) -> bool:
    """
    Whether the end of the data cuts the segment.
    """
    return len(self.data) < self.size

class Jpeg (reader)

A JPEG image, which begins with its start-of-image marker and ends with its end-of-image marker or with the end of the data; the data that follows the end-of-image marker is not part of the structure. The junk of the image is the data in front of a marker that is not fill bytes, and the rest of the data when it ends inside a marker or inside the length of a segment.

Expand source code Browse git

class Jpeg(Struct):
    """
    A JPEG image, which begins with its start-of-image marker and ends with its end-of-image marker
    or with the end of the data; the data that follows the end-of-image marker is not part of the
    structure. The junk of the image is the data in front of a marker that is not fill bytes, and
    the rest of the data when it ends inside a marker or inside the length of a segment.
    """
    def __init__(self, reader: StructReader):
        self.streams: list[JpegStream] = []
        self.junk: list[memoryview] = []
        self.sof = None
        self.scandata: list[memoryview] = []
        self.scans: list[JpegStream] = []
        self.comments: list[JpegStream] = []
        self.meta: list[JpegStream] = []

        reader.bigendian = True

        if reader.peek(2) != B'\xFF\xD8':
            raise ValueError('The data does not begin with a start-of-image marker.')

        while not reader.eof:
            start = reader.tell()
            marker = _MARKER.search(reader.peek())
            if marker is None or marker.start() > 0:
                self.junk.append(reader.read(None if marker is None else marker.start()))
                continue
            try:
                stream = JpegStream(reader)
            except EOF:
                reader.seekset(start)
                self.junk.append(reader.read())
                break
            self.streams.append(stream)
            if (t := stream.type).is_frame:
                if self.sof is not None:
                    raise ValueError('Duplicate SOF Stream in File.')
                if not stream.truncated:
                    self.sof = JpegSOF.Parse(stream.data)
            elif t == JpegMarker.StartOfScan:
                self.scans.append(stream)
            elif t == JpegMarker.Comment:
                self.comments.append(stream)
            elif t in range(JpegMarker.App00, JpegMarker.App15 + 1):
                self.meta.append(stream)
            elif t == JpegMarker.EndOfImage:
                break

    @property
    def truncated(self) -> bool:
        """
        Whether the data ends before the end-of-image marker.
        """
        return self.streams[-1].type != JpegMarker.EndOfImage

Ancestors

• Struct
• typing.Generic
• collections.abc.Buffer

Static methods

def Parse(reader, *args, **kwargs)

Instance variables

var truncated

Whether the data ends before the end-of-image marker.

Expand source code Browse git

@property
def truncated(self) -> bool:
    """
    Whether the data ends before the end-of-image marker.
    """
    return self.streams[-1].type != JpegMarker.EndOfImage

class jpeg (*paths, exclude=None, list=False, join_path=False, drop_path=False, fuzzy=0, exact=False, regex=False, path=b'path')

Extract the raw segments from a JPG image. Parses JFIF/EXIF markers to recover embedded thumbnails, metadata, and comment segments. Data appended after the end of the image, such as a hidden payload, is extracted as the overlay. Data between the segments that belongs to none of them is extracted as junk. A truncated image, such as one carved from a larger file, gives everything up to the cut, and a segment that the cut divides gives the part in front of it.

Expand source code Browse git

class jpeg(PathExtractorUnit):
    """
    Extract the raw segments from a JPG image. Parses JFIF/EXIF markers to recover embedded
    thumbnails, metadata, and comment segments. Data appended after the end of the image, such as
    a hidden payload, is extracted as the overlay. Data between the segments that belongs to none
    of them is extracted as junk. A truncated image, such as one carved from a larger file, gives
    everything up to the cut, and a segment that the cut divides gives the part in front of it.
    """
    def unpack(self, data):
        view = memoryview(data)
        jpg = Jpeg.Parse(view)
        if jpg.junk:
            self.log_warn('the image contains data that belongs to none of its segments')
        if jpg.truncated:
            self.log_warn('the data ends before the end of the image')
        for k, stream in enumerate(jpg.streams):
            yield UnpackResult(F'streams/{k}.{stream.type.name}', stream.__buffer__(0))
        for k, comment in enumerate(jpg.comments):
            yield UnpackResult(F'comments/{k}', comment.data)
        for k, scan in enumerate(jpg.scans):
            yield UnpackResult(F'scans/{k}', scan.scan)
        for k, meta in enumerate(jpg.meta):
            extension = {
                JpegMarker.App00: '.jfif',
                JpegMarker.App01: '.exif',
                JpegMarker.App02: '.iccp',
            }.get((t := meta.type), '')
            yield UnpackResult(F'meta/{k}.{t.name.lower()}{extension}', meta.data)
        if sof := jpg.sof:
            yield UnpackResult('meta/dimensions.json', json.dumps(struct_to_json(sof)))
        for k, junk in enumerate(jpg.junk):
            yield UnpackResult(F'junk/{k}', junk)
        if overlay := view[len(jpg):]:
            yield UnpackResult('overlay', overlay)

    @classmethod
    def handles(cls, data) -> bool:
        return get_image_format(data) == Fmt.JPG

Ancestors

• PathExtractorUnit
• Unit
• UnitBase
• Entry

Subclasses

• jpeg

Class variables

var reverse

The type of the None singleton.

Methods

def unpack(self, data)

Expand source code Browse git

def unpack(self, data):
    view = memoryview(data)
    jpg = Jpeg.Parse(view)
    if jpg.junk:
        self.log_warn('the image contains data that belongs to none of its segments')
    if jpg.truncated:
        self.log_warn('the data ends before the end of the image')
    for k, stream in enumerate(jpg.streams):
        yield UnpackResult(F'streams/{k}.{stream.type.name}', stream.__buffer__(0))
    for k, comment in enumerate(jpg.comments):
        yield UnpackResult(F'comments/{k}', comment.data)
    for k, scan in enumerate(jpg.scans):
        yield UnpackResult(F'scans/{k}', scan.scan)
    for k, meta in enumerate(jpg.meta):
        extension = {
            JpegMarker.App00: '.jfif',
            JpegMarker.App01: '.exif',
            JpegMarker.App02: '.iccp',
        }.get((t := meta.type), '')
        yield UnpackResult(F'meta/{k}.{t.name.lower()}{extension}', meta.data)
    if sof := jpg.sof:
        yield UnpackResult('meta/dimensions.json', json.dumps(struct_to_json(sof)))
    for k, junk in enumerate(jpg.junk):
        yield UnpackResult(F'junk/{k}', junk)
    if overlay := view[len(jpg):]:
        yield UnpackResult('overlay', overlay)

Inherited members

• PathExtractorUnit:
  • CustomJoinBehaviour
  • CustomPathSeparator
  • FilterEverything
  • Requires
  • act
  • assemble
  • carved
  • close
  • codec
  • console
  • filter
  • finish
  • handles
  • is_quiet
  • is_reversible
  • isatty
  • labelled
  • leniency
  • log_always
  • log_debug
  • log_detach
  • log_fail
  • log_info
  • log_level
  • log_warn
  • logger
  • name
  • nozzle
  • optional_dependencies
  • read
  • read1
  • required_dependencies
  • reset
  • run
  • source
  • superinit
• UnitBase:
  • process