build(deps): bump github.com/gabriel-vasile/mimetype

Bumps [github.com/gabriel-vasile/mimetype](https://github.com/gabriel-vasile/mimetype) from 1.4.13 to 1.4.15.
- [Release notes](https://github.com/gabriel-vasile/mimetype/releases)
- [Commits](https://github.com/gabriel-vasile/mimetype/compare/v1.4.13...v1.4.15)

---
updated-dependencies:
- dependency-name: github.com/gabriel-vasile/mimetype
  dependency-version: 1.4.15
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
This commit is contained in:
dependabot[bot]
2026-08-03 07:31:56 +00:00
committed by Ralf Haferkamp
parent e63f1d542a
commit 639e3657cd
27 changed files with 1408 additions and 233 deletions

2
go.mod
View File

@@ -18,7 +18,7 @@ require (
github.com/davidbyttow/govips/v2 v2.18.0
github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
github.com/gabriel-vasile/mimetype v1.4.13
github.com/gabriel-vasile/mimetype v1.4.15
github.com/ggwhite/go-masker v1.1.0
github.com/go-chi/chi/v5 v5.3.1
github.com/go-chi/render v1.0.3

4
go.sum
View File

@@ -350,8 +350,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
github.com/gdexlab/go-render v1.0.1 h1:rxqB3vo5s4n1kF0ySmoNeSPRYkEsyHgln4jFIQY7v0U=
github.com/gdexlab/go-render v1.0.1/go.mod h1:wRi5nW2qfjiGj4mPukH4UV0IknS1cHD4VgFTmJX5JzM=
github.com/getkin/kin-openapi v0.13.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw=

View File

@@ -7,6 +7,12 @@ linters:
exclusions:
presets:
- std-error-handling
rules:
# Test fixtures construct CDF binary blobs from known small constants, so
# gosec's integer-overflow checks (G115) add no value there.
- path: internal/cdf/cdf_test\.go
linters:
- gosec
enable:
- gosec # Detects security problems.
# Keep all extras disabled for now to focus on the integer overflow problem.
@@ -31,6 +37,7 @@ linters:
- unused
- usestdlibvars # Detects the possibility to use variables/constants from the Go standard library.
- usetesting # Reports uses of functions with replacement inside the testing package.
- asciicheck # https://daniel.haxx.se/blog/2025/05/16/detecting-malicious-unicode/
settings:
govet:
disable:

View File

@@ -13,8 +13,8 @@
<a href="https://pkg.go.dev/github.com/gabriel-vasile/mimetype">
<img alt="Go Reference" src="https://pkg.go.dev/badge/github.com/gabriel-vasile/mimetype.svg">
</a>
<a href="https://goreportcard.com/report/github.com/gabriel-vasile/mimetype">
<img alt="Go report card" src="https://goreportcard.com/badge/github.com/gabriel-vasile/mimetype">
<a href="https://codecov.io/gh/gabriel-vasile/mimetype">
<img alt="Code coverage" src="https://codecov.io/gh/gabriel-vasile/mimetype/graph/badge.svg">
</a>
<a href="LICENSE">
<img alt="License" src="https://img.shields.io/badge/License-MIT-green.svg">
@@ -103,3 +103,10 @@ shows which file formats are most often misidentified and can help prioritise.
When submitting a PR for detection of a new file format, please make sure to
add a record to the list of testcases in [mimetype_test.go](mimetype_test.go).
For complex files a record can be added in the [testdata](testdata) directory.
Code contributions must respect following rules:
- code must be test covered
- code must be formatted using the `gofmt` tool
- exported names must be documented
**Important**: By submitting a pull request, you agree to allow the project
owner to license your work under the same license as that used by the project.

View File

@@ -0,0 +1 @@
comment: false

View File

@@ -0,0 +1,667 @@
// Package cdf implements parsing of CDF (OLE2) files. It is greatly inspired
// by src/readcdf.c from libmagic. One difference is this implementation is
// permissive of truncated inputs. See readLimit in mimetype.go for the
// reason why truncated inputs need to be handled.
// http://sc.openoffice.org/compdocfileformat.pdf
package cdf
import (
"bytes"
"encoding/binary"
"github.com/gabriel-vasile/mimetype/internal/scan"
)
type CDFType int8
const (
CDFTypeGeneric CDFType = iota
CDFTypeInstaller
CDFTypeDoc
CDFTypePpt
CDFTypeXls
CDFTypeMsg
)
// Detect parses raw as a CDF (OLE2) compound file and returns the document type
// it contains. It returns CDFTypeGeneric for input that is not a CDF file or
// whose type cannot be narrowed down.
func Detect(raw []byte) CDFType {
if len(raw) < 512 {
return CDFTypeGeneric
}
var c cdf
if !parse(raw, &c) {
return CDFTypeGeneric
}
return c.detect()
}
// cdf holds everything we need from a CDF file to do detection.
type cdf struct {
data []byte
secSize int
shortSecSize int
minStdStream uint32
satSecs int32s // list of SAT sector ids; usually a sub-slice of raw input
satEntries int // number of valid SAT entries reachable through satSecs
firstSSAT int32
dirRaw []byte // directory stream bytes (entries are decoded on demand)
sst []byte // short-stream pool (root storage's stream)
sstBuilt bool // whether sst was already loaded (it is loaded lazily)
rootStreamFirst int32 // first sector of the root storage short-stream pool
rootStreamSize uint32 // size of the root storage short-stream pool
rootStorageUUID []byte
}
// parse reads the entire on-disk structure required for type detection. It
// returns true on success and false if the header does not look like a CDF file.
// Truncated or partially malformed bodies are tolerated: sector reads degrade
// to whatever could be collected so detection can still succeed from partial data.
func parse(raw []byte, c *cdf) bool {
if len(raw) < 512 || binary.LittleEndian.Uint64(raw) != cdfMagic {
return false
}
secP2 := binary.LittleEndian.Uint16(raw[30:32])
shortP2 := binary.LittleEndian.Uint16(raw[32:34])
if secP2 > 20 || shortP2 > 20 {
return false
}
c.data = raw
c.secSize = 1 << secP2
c.shortSecSize = 1 << shortP2
c.minStdStream = binary.LittleEndian.Uint32(raw[56:60])
if c.secSize < dirEntrySize {
return false
}
firstDirSec := readSecID(raw[48:52])
c.firstSSAT = readSecID(raw[60:64])
firstMSAT := readSecID(raw[68:72])
nMSAT := binary.LittleEndian.Uint32(raw[72:76])
masterSAT := int32s{b: raw[76 : 76+4*masterSATSize]}
c.buildSAT(masterSAT, firstMSAT, nMSAT)
c.dirRaw = c.readLong(firstDirSec, 0)
c.rootStreamFirst = -1
var d dirEntry
for i, n := 0, c.dirLen(); i < n; i++ {
c.dirAt(i, &d)
if d.typ != dirTypeRootStorage || d.streamFirst < 0 {
continue
}
c.rootStorageUUID = d.storageUUID[:]
// Record where the short-stream pool lives; it is loaded lazily by
// shortStream the first time a short stream is actually read.
c.rootStreamFirst = d.streamFirst
c.rootStreamSize = d.size
break
}
return true
}
func (c *cdf) detect() CDFType {
for _, name := range []string{"\x05SummaryInformation", "\x05DocumentSummaryInformation"} {
if t, ok := c.detectFromSummary(name); ok {
return t
}
}
var d dirEntry
for i, n := 0, c.dirLen(); i < n; i++ {
c.dirAt(i, &d)
if t, ok := lookupSection(d.nameBytes(), d.typ); ok {
return t
}
}
return CDFTypeGeneric
}
// detectFromSummary inspects a (Doc)SummaryInformation stream and tries to
// derive a CDFType from the root-storage CLSID, the property NameOfApplication,
// and finally the names of sibling user streams.
func (c *cdf) detectFromSummary(streamName string) (CDFType, bool) {
if c.rootStorageUUID != nil && bytes.Equal(c.rootStorageUUID, msiCLSID) {
return CDFTypeInstaller, true
}
raw, ok := c.userStream(streamName)
if !ok {
return CDFTypeGeneric, false
}
if app := summaryAppName(raw); len(app) > 0 {
if t, ok := lookupSubstring(app, app2type); ok {
return t, true
}
}
for i, n := 0, c.dirLen(); i < n; i++ {
var d dirEntry
c.dirAt(i, &d)
if d.nameLen == 0 {
continue
}
if t, ok := lookupSubstring(d.nameBytes(), name2type); ok {
return t, true
}
}
return CDFTypeGeneric, true
}
const (
cdfMagic uint64 = 0xE11AB1A1E011CFD0
dirTypeUserStorage = 1
dirTypeUserStream = 2
dirTypeRootStorage = 5
dirEntrySize = 128
masterSATSize = 109 // first 109 SAT secids live in the file header
)
// dirEntry is a single CDF directory record. The UTF-16LE name is pre-decoded
// into an inline ASCII buffer at parse time, avoiding a per-entry heap
// allocation while keeping comparisons trivial. CDF names are at most 32
// UTF-16 code units, so 32 bytes always suffice.
type dirEntry struct {
name [32]byte
nameLen uint8
typ uint8
streamFirst int32
size uint32
storageUUID [16]byte
}
// nameBytes returns the decoded ASCII name without copying.
func (d *dirEntry) nameBytes() []byte { return d.name[:d.nameLen] }
func (c *cdf) ssatAt(i int32) int32 {
for sid := c.firstSSAT; sid >= 0; {
if int(sid) >= c.satLen() {
break // SAT is truncated; stop collecting
}
buf, ok := c.sector(sid)
if !ok {
break
}
lbuf := int32(len(buf) / 4) //nolint:gosec // anything divided by 4 fits int32
if i < lbuf {
return int32(binary.LittleEndian.Uint32(buf[4*i:])) //nolint:gosec // intentional two's-complement reinterpretation of a sector id
}
i -= lbuf
sid = c.satAt(sid)
}
return -1
}
// shortStream returns the root storage short-stream pool, loading it on first
// use. Detection often finishes (e.g. via the root CLSID or a long-stream
// summary) without ever reading a short stream, so building this eagerly would
// be wasted work.
func (c *cdf) shortStream() []byte {
if !c.sstBuilt {
c.sstBuilt = true
if c.rootStreamFirst >= 0 {
c.sst = c.readLong(c.rootStreamFirst, c.rootStreamSize)
}
}
return c.sst
}
// int32s works like a slice of LE int32 and is backed by a slice of bytes.
// int32s could very well be type int32s []byte, but that would mean
// len function can be called on it. We don't want that, we always want to use
// the len method.
type int32s struct {
b []byte
}
func (b int32s) at(i int) int32 {
//nolint:gosec // intentional two's-complement reinterpretation of a sector id
return int32(binary.LittleEndian.Uint32(b.b[4*i:]))
}
func (b int32s) len() int {
return len(b.b) / 4
}
// readSecID reinterprets four little-endian bytes as a signed sector id.
// Every 32-bit pattern is a valid id (values >= 0 are sector numbers,
// negatives are CDF sentinels such as -2 end-of-chain), so the conversion is
// an intentional two's-complement reinterpretation rather than an overflow.
func readSecID(b []byte) int32 {
return int32(binary.LittleEndian.Uint32(b)) //nolint:gosec // intentional two's-complement reinterpretation
}
// satLen is the number of sector ids reachable through the SAT.
func (c *cdf) satLen() int { return c.satEntries }
// satAt returns the i-th sector id from the SAT. Callers must ensure
// i < satLen(). The SAT is not materialized; the entry is fetched directly
// from the input by translating i into (SAT sector index, entry offset).
func (c *cdf) satAt(i int32) int32 {
perSec := c.secSize / 4
secIdx := int(i) / perSec
entryIdx := int(i) % perSec
secID := c.satSecs.at(secIdx)
off := c.secSize*(1+int(secID)) + 4*entryIdx
return readSecID(c.data[off:])
}
// sector returns the bytes of long sector secid. If the file is truncated
// inside the requested sector the result is the available bytes (no padding).
// If the sector starts past EOF or secid is negative, then ok is false.
func (c *cdf) sector(secid int32) (_ []byte, ok bool) {
if secid < 0 {
return nil, false
}
off := int64(c.secSize) * (1 + int64(secid))
if off >= int64(len(c.data)) {
return nil, false
}
// The returned sector might be truncated,
// but we still return it as best effort.
end := min(off+int64(c.secSize), int64(len(c.data)))
// If not even one int32 fits, then fail.
if end-off < 4 {
return nil, false
}
return c.data[off:end], true
}
func (c *cdf) sectorIDs(secid int32) (int32s, bool) {
buf, ok := c.sector(secid)
if !ok {
return int32s{}, ok
}
return int32s{b: buf}, true
}
// buildSAT records the list of SAT sector ids from the master-SAT (header)
// plus any extension blocks chained via firstMSAT. The SAT itself is not
// materialized: satAt computes the requested entry directly from c.data via
// satSecs. In the common case (no extension chain) satSecs is a zero-copy
// sub-slice of the input header.
func (c *cdf) buildSAT(masterSAT int32s, firstMSAT int32, nMSAT uint32) {
// Fast path: no extension chain. masterSAT is already a sub-slice of raw
// input; reuse it directly.
if firstMSAT < 0 || nMSAT == 0 {
c.satSecs = masterSAT
c.satEntries = c.computeSATLen()
return
}
// Slow path: gather sector ids from the header plus the extension chain
// into a fresh buffer. Even here we only allocate space for ids (4 bytes
// each), not the full SAT contents.
maxIDs := len(c.data)/c.secSize + 1
buf := make([]byte, 0, 4*masterSATSize)
for i := 0; i < masterSAT.len(); i++ {
if masterSAT.at(i) < 0 {
break
}
buf = append(buf, masterSAT.b[4*i:4*i+4]...)
}
perSec := c.secSize/4 - 1
mid := firstMSAT
chain:
for j := uint32(0); j < nMSAT && mid >= 0; j++ {
msa, ok := c.sectorIDs(mid)
if !ok {
break
}
for k := 0; k < perSec; k++ {
if k >= msa.len() || msa.at(k) < 0 {
break chain
}
buf = append(buf, msa.b[4*k:4*k+4]...)
if len(buf)/4 > maxIDs {
break chain // cyclic MSAT chain; stop allocating
}
}
if perSec >= msa.len() {
break // no next-MSAT pointer available
}
mid = msa.at(perSec)
}
c.satSecs = int32s{b: buf}
c.satEntries = c.computeSATLen()
}
// computeSATLen walks satSecs and counts how many SAT entries are actually
// reachable in c.data, stopping at the first sentinel id or sector that is not
// fully present in the file.
func (c *cdf) computeSATLen() int {
perSec := c.secSize / 4
total := 0
for i := 0; i < c.satSecs.len(); i++ {
sec := c.satSecs.at(i)
if sec < 0 {
break
}
off := int64(c.secSize) * (1 + int64(sec))
if off >= int64(len(c.data)) {
break
}
avail := int64(len(c.data)) - off
if avail >= int64(c.secSize) {
total += perSec
continue
}
total += int(avail / 4)
break
}
return total
}
// readLong reads a long-sector chain starting at sid. If length > 0 the
// result is truncated to that many bytes. On truncation or any other failure
// it returns whatever sectors were readable.
func (c *cdf) readLong(sid int32, length uint32) []byte {
// Fast path: when the chain is a single physically contiguous run of
// sectors (the common case for the directory and summary streams) the data
// is already laid out sequentially in the input, so return a sub-slice of
// it instead of allocating a buffer and copying every sector.
if sid >= 0 {
maxSec := len(c.data)/c.secSize + 1
n, s := 0, sid
contiguous := true
for s >= 0 {
if int(s) >= c.satLen() {
break // SAT truncated; what remains is still contiguous
}
n++
if n > maxSec {
contiguous = false // cyclic chain; let the slow path guard it
break
}
next := c.satAt(s)
if next >= 0 && int64(next) != int64(s)+1 {
contiguous = false
break
}
s = next
}
if contiguous {
off64 := int64(c.secSize) * (1 + int64(sid))
if off64 >= int64(len(c.data)) {
return nil
}
end64 := min(off64+int64(n)*int64(c.secSize), int64(len(c.data)))
out := c.data[off64:end64]
if length > 0 && int64(length) < int64(len(out)) {
out = out[:length]
}
return out
}
}
// Slow path: gather a fragmented chain into a fresh buffer. Real-world
// writers (MSI builders, edited Office documents) routinely produce
// non-contiguous directory and stream chains, so this fallback is required
// for correct detection on those files.
maxBytes := len(c.data)
out := make([]byte, 0, c.secSize)
for sid >= 0 {
if int(sid) >= c.satLen() {
break // SAT truncated; return what we have
}
buf, ok := c.sector(sid)
if !ok {
break
}
out = append(out, buf...)
if len(out) >= maxBytes {
break // chain longer than the file: cyclic SAT, stop
}
sid = c.satAt(sid)
}
if length > 0 && int64(length) < int64(len(out)) {
out = out[:length]
}
return out
}
// readShort reads a short-sector chain at sid by indexing into the short-stream
// pool. On truncation or if the pool is unavailable it returns whatever was
// readable (possibly nil).
func (c *cdf) readShort(sid int32, length uint32) []byte {
sst := c.shortStream()
if sst == nil {
return nil
}
// TODO: anyway to avoid allocating and copying the bytes?
out := make([]byte, 0, c.shortSecSize)
for sid >= 0 {
off64 := int64(sid) * int64(c.shortSecSize)
if off64+int64(c.shortSecSize) > int64(len(sst)) {
break // short-stream pool truncated or sid out of range
}
off := int(off64)
out = append(out, sst[off:off+c.shortSecSize]...)
if len(out) >= len(sst) {
break // chain longer than the pool: cyclic SSAT, stop
}
sid = c.ssatAt(sid)
}
if length > 0 && int64(length) < int64(len(out)) {
out = out[:length]
}
return out
}
// readChain dispatches to the long or short reader depending on stream size.
func (c *cdf) readChain(sid int32, length uint32) []byte {
if length < c.minStdStream && c.rootStreamFirst >= 0 {
return c.readShort(sid, length)
}
return c.readLong(sid, length)
}
// dirLen returns the number of directory entries in dirRaw.
func (c *cdf) dirLen() int { return len(c.dirRaw) / dirEntrySize }
// dirAt decodes the i-th directory entry into *out. Callers must ensure
// i < dirLen(). The UTF-16LE name is decoded into out.name, ASCII-style,
// stopping at the first NUL.
func (c *cdf) dirAt(i int, out *dirEntry) {
raw := c.dirRaw[i*dirEntrySize:]
nameLen := min(int(binary.LittleEndian.Uint16(raw[64:])), 64)
k := uint8(0)
for j := 0; j < nameLen/2; j++ {
// Names are ASCII; keep the low byte of each little-endian UTF-16
// code unit and stop at the first NUL.
lo, hi := raw[2*j], raw[2*j+1]
if lo == 0 && hi == 0 {
break
}
out.name[k] = lo
k++
}
out.nameLen = k
out.typ = raw[66]
out.streamFirst = readSecID(raw[116:120])
out.size = binary.LittleEndian.Uint32(raw[120:])
copy(out.storageUUID[:], raw[80:96])
}
// userStream finds a user stream by name and returns its bytes.
func (c *cdf) userStream(name string) ([]byte, bool) {
var d dirEntry
for i, n := 0, c.dirLen(); i < n; i++ {
c.dirAt(i, &d)
if d.typ == dirTypeUserStream && string(d.nameBytes()) == name {
buf := c.readChain(d.streamFirst, d.size)
if buf == nil {
return nil, false
}
return buf, true
}
}
return nil, false
}
const (
propIDNameOfApplication = 0x12
typeMask = 0x0fff
typeVector = 0x1000
typeStringASCII = 0x1e
typeStringWide = 0x1f
sectionDeclOffset = 0x1c // section declaration in property-set header
)
// summaryAppName parses a (Doc)SummaryInformation stream and returns the
// value of property NameOfApplication (0x12) as printable ASCII, or nil if
// not present or the stream is malformed. This is the only summary property
// the detection logic ever consults.
func summaryAppName(stream []byte) []byte {
if len(stream) < sectionDeclOffset+20 {
return nil
}
sdOff := binary.LittleEndian.Uint32(stream[sectionDeclOffset+16:])
if uint64(sdOff)+8 > uint64(len(stream)) {
return nil
}
section := stream[sdOff:]
shLen := binary.LittleEndian.Uint32(section[0:])
nProps := binary.LittleEndian.Uint32(section[4:])
if uint64(shLen) > uint64(len(section)) || nProps > 1<<16 || 8+8*nProps > shLen {
return nil
}
for i := uint32(0); i < nProps; i++ {
base := 8 + 8*i
id := binary.LittleEndian.Uint32(section[base:])
if id != propIDNameOfApplication {
continue
}
off := binary.LittleEndian.Uint32(section[base+4:])
if uint64(off)+8 > uint64(shLen) {
return nil
}
typ := binary.LittleEndian.Uint32(section[off:])
if typ&typeVector != 0 {
return nil
}
step := uint32(0)
switch typ & typeMask {
case typeStringASCII:
step = 1
case typeStringWide:
step = 2
default:
return nil
}
slen := binary.LittleEndian.Uint32(section[off+4:])
start := uint64(off) + 8
end := start + uint64(slen)*uint64(step)
if end > uint64(shLen) {
return nil
}
return printableLowBytes(section[start:end], int(step))
}
return nil
}
// printableLowBytes copies the printable low byte of each step-byte unit
// in b, stopping at the first NUL.
func printableLowBytes(b []byte, step int) []byte {
out := make([]byte, 0, len(b)/step)
for i := 0; i+step <= len(b); i += step {
c := b[i]
if c == 0 {
break
}
if c >= 0x20 && c < 0x7f {
out = append(out, c)
}
}
return out
}
// pattern is a case-insensitive substring → CDFType mapping. Entries are
// tested in order; first match wins. needle is stored upper-cased so it can be
// matched case-insensitively by scan.Bytes.Search with scan.IgnoreCase.
type pattern struct {
needle []byte
typ CDFType
}
// app2type maps NameOfApplication values to CDFTypes.
// Mirrors app2mime[] in libmagic. Needles are upper-cased for case-insensitive
// matching via scan.IgnoreCase.
var app2type = []pattern{
{[]byte("WORD"), CDFTypeDoc},
{[]byte("EXCEL"), CDFTypeXls},
{[]byte("POWERPOINT"), CDFTypePpt},
{[]byte("ADVANCED INSTALLER"), CDFTypeInstaller},
{[]byte("INSTALLSHIELD"), CDFTypeInstaller},
{[]byte("MICROSOFT PATCH COMPILER"), CDFTypeInstaller},
{[]byte("NANT"), CDFTypeInstaller},
{[]byte("WINDOWS INSTALLER"), CDFTypeInstaller},
}
// name2type maps directory entry names to CDFTypes.
// Mirrors name2mime[] in libmagic. Needles are upper-cased for case-insensitive
// matching via scan.IgnoreCase.
var name2type = []pattern{
{[]byte("BOOK"), CDFTypeXls},
{[]byte("WORKBOOK"), CDFTypeXls},
{[]byte("WORDDOCUMENT"), CDFTypeDoc},
{[]byte("POWERPOINT"), CDFTypePpt},
{[]byte("DIGITALSIGNATURE"), CDFTypeInstaller},
}
// lookupSubstring returns the CDFType for the first entry in t whose needle
// is a case-insensitive substring of v. Mirrors C's strcasestr semantics
// under the C locale. It allocates nothing: scan.IgnoreCase matches the
// upper-cased needle against input of either case.
func lookupSubstring(v []byte, t []pattern) (CDFType, bool) {
s := scan.Bytes(v)
for _, p := range t {
if i, _ := s.Search(p.needle, scan.IgnoreCase); i != -1 {
return p.typ, true
}
}
return CDFTypeGeneric, false
}
// msiCLSID is the Microsoft Installer root-storage CLSID, in on-disk byte
// order (cdf_directory_t.d_storage_uuid stores two little-endian uint64s).
var msiCLSID = []byte{
0x84, 0x10, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00,
0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46,
}
// section is a (directory entry name, type) → CDFType mapping.
type section struct {
name string
typ uint8
cdf CDFType
}
// sectionTypes maps distinctive directory entries to CDFTypes — a flattened
// equivalent of sectioninfo[] in libmagic. Used as a fallback when no
// SummaryInformation stream is present. A slice (rather than a map) lets
// lookupSection compare entry names without allocating a string key.
var sectionTypes = []section{
// libmagic uses application/encrypted, but that is not a registered media type.
// For now, we skip identifying that and fall-back on CDFTypeGeneric
// {"EncryptedPackage", dirTypeUserStream, CDFTypeEncrypted},
// {"EncryptedSummary", dirTypeUserStream, CDFTypeEncrypted},
{"Book", dirTypeUserStream, CDFTypeXls},
{"Workbook", dirTypeUserStream, CDFTypeXls},
{"WordDocument", dirTypeUserStream, CDFTypeDoc},
{"PowerPoint Document", dirTypeUserStream, CDFTypePpt},
{"__properties_version1.0", dirTypeUserStream, CDFTypeMsg},
{"__recip_version1.0_#00000000", dirTypeUserStorage, CDFTypeMsg},
}
// lookupSection returns the CDFType for a directory entry whose name and type
// match a sectionTypes entry exactly. The string(name) == comparison is
// optimized by the compiler to avoid allocating.
func lookupSection(name []byte, typ uint8) (CDFType, bool) {
for _, s := range sectionTypes {
if s.typ == typ && string(name) == s.name {
return s.cdf, true
}
}
return CDFTypeGeneric, false
}

View File

@@ -84,19 +84,8 @@ func FromPlain(content []byte) string {
break
}
}
hasHighBit := false
for _, c := range content {
if c >= 0x80 {
hasHighBit = true
break
}
}
if hasHighBit && utf8.Valid(content) {
return "utf-8"
}
// ASCII is a subset of UTF8. Follow W3C recommendation and replace with UTF8.
if ascii(origContent) {
if utf8.Valid(content) {
return "utf-8"
}
@@ -123,15 +112,6 @@ func latin(content []byte) string {
return "iso-8859-1"
}
func ascii(content []byte) bool {
for _, b := range content {
if textChars[b] != T {
return false
}
}
return true
}
// FromXML returns the charset of an XML document. It relies on the XML
// header <?xml version="1.0" encoding="UTF-8"?> and falls back on the plain
// text content.

View File

@@ -12,10 +12,10 @@ import (
type Parser struct {
comma byte
comment byte
s scan.Bytes
s *scan.Bytes
}
func NewParser(comma, comment byte, s scan.Bytes) *Parser {
func NewParser(comma, comment byte, s *scan.Bytes) *Parser {
return &Parser{
comma: comma,
comment: comment,
@@ -55,7 +55,7 @@ func (r *Parser) CountFields(collectIndexes bool) (fields int, fieldPos []int, h
if finished {
return 0, nil, false
}
finished = len(r.s) == 0 && len(line) == 0
finished = len(*r.s) == 0 && len(line) == 0
if len(line) == lengthNL(line) {
line = nil
continue // Skip empty lines.

View File

@@ -10,6 +10,7 @@ const (
QueryGeo = "geo"
QueryHAR = "har"
QueryGLTF = "gltf"
QueryCDX = "cdx"
maxRecursion = 4096
)
@@ -40,6 +41,10 @@ var queries = map[string][]query{
SearchPath: [][]byte{[]byte("asset"), []byte("version")},
SearchVals: [][]byte{[]byte(`"1.0"`), []byte(`"2.0"`)},
}},
QueryCDX: {{
SearchPath: [][]byte{[]byte("bomFormat")},
SearchVals: [][]byte{[]byte(`"CycloneDX"`)},
}},
}
var parserPool = sync.Pool{

View File

@@ -3,6 +3,8 @@ package magic
import (
"bytes"
"encoding/binary"
"github.com/gabriel-vasile/mimetype/internal/mp3"
)
// Flac matches a Free Lossless Audio Codec file.
@@ -51,32 +53,83 @@ func AAC(raw []byte, _ uint32) bool {
return len(raw) > 1 && ((raw[0] == 0xFF && raw[1] == 0xF1) || (raw[0] == 0xFF && raw[1] == 0xF9))
}
// Mp3 matches an mp3 file.
func Mp3(raw []byte, limit uint32) bool {
// MP3 matches a .mp3 file.
func MP3(raw []byte, limit uint32) bool {
if len(raw) < 3 {
return false
}
if bytes.HasPrefix(raw, []byte("ID3")) {
// MP3s with an ID3v2 tag will start with "ID3"
// ID3v1 tags, however appear at the end of the file.
// Any ID3v2 is reported as MP3. Not entirely correct, but the mimesniff
// standard says so. https://mimesniff.spec.whatwg.org/#matching-an-audio-or-video-type-pattern
// Despite the standard only checking for "ID3", we do more validations to
// avoid false positives.
if id3v2(raw) {
return true
}
// Match MP3 files without tags
// If no ID3v2 tag found, then we will look for MP3 frames, but:
// a. Layer III files are a lot more prevalent than Layer I and II.
// b. Layer I frame header has looser constraints than the others: many files
// with regularly repeating 0xFFFF bytes can be misidentified as MP3.
// c. MP3 files are composed of individual frames and those frames can have
// leading garbage bytes: if we want to find all valid MP3s, we have to do a
// linear search. #775, #310
// d. There are file formats that contain MP3s inside: .mo3 and .swa
//
// Given a, b, c and d, this code:
// - initially tries to match by first two bytes in header
// - checks for .mo3 and .swa and disqualifies them
// - does linear search for Layer III
switch binary.BigEndian.Uint16(raw[:2]) & 0xFFFE {
case 0xFFFA:
// MPEG ADTS, layer III, v1
return true
case 0xFFF2:
// MPEG ADTS, layer III, v2
return true
case 0xFFE2:
// MPEG ADTS, layer III, v2.5
case 0xFFFA, 0xFFF2, 0xFFE2, // layer III: v1, v2, v2.5
0xFFFC, 0xFFF4, // layer II: v1, v2
0xFFF5: // layer I: v2
return true
}
// http://lclevy.free.fr/mo3/
if bytes.HasPrefix(raw, []byte("MO3")) {
return false
}
return false
// From PRONOM:
// Macromedia licensed the MP3 technology in 1995 to use in their Shockwave
// product. .swa or Shockwave Audio was originally added as a free plugin
// (Xtras) to SoundEdit 16 to export AIFF files to .swa.
// There is no media type assigned for .swa.
if bytes.HasPrefix(raw, []byte{0x00, 0x00, 0x01, 0x40, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00}) {
return false
}
_, size := mp3.ExtractFrame(raw)
return size > 0
}
// Based on https://id3.org/Developer%20Information.
func id3v2(raw []byte) bool {
if len(raw) < 10 || !bytes.HasPrefix(raw, []byte("ID3")) {
return false
}
if raw[3] < 2 || raw[3] > 4 { // Version: ID3v2.2 - ID3v2.4.
return false
}
if raw[4] != 0 { // Revision is 0 for all versions.
return false
}
// v2.2 uses 2 bits, v2.3 uses 3 bits and v2.4 uses 4.
// For all versions least significant 4 bits should be 0
if raw[5]&0b1111 != 0 {
return false
}
// Size bytes are synchsafe: most significant bit always 0.
if raw[6]&0x80 != 0 || raw[7]&0x80 != 0 || raw[8]&0x80 != 0 || raw[9]&0x80 != 0 {
return false
}
size := uint32(raw[6])<<21 | uint32(raw[7])<<14 | uint32(raw[8])<<7 | uint32(raw[9])
// Disallow too big frames, let's say 10MB.
return size > 0 && size < 10*1024*1024
}
// Wav matches a Waveform Audio File Format file.

View File

@@ -4,6 +4,7 @@ import (
"bytes"
"debug/macho"
"encoding/binary"
"slices"
)
// Lnk matches Microsoft lnk binary format.
@@ -117,13 +118,7 @@ func Dbf(raw []byte, limit uint32) bool {
0x02, 0x03, 0x04, 0x05, 0x30, 0x31, 0x32, 0x42, 0x62, 0x7B, 0x82,
0x83, 0x87, 0x8A, 0x8B, 0x8E, 0xB3, 0xCB, 0xE5, 0xF5, 0xF4, 0xFB,
}
for _, b := range dbfTypes {
if raw[0] == b {
return true
}
}
return false
return slices.Contains(dbfTypes, raw[0])
}
// ElfObj matches an object file.
@@ -229,3 +224,63 @@ func TzIf(raw []byte, limit uint32) bool {
// Version has to be NUL (0x00), '2' (0x32) or '3' (0x33).
return raw[4] == 0x00 || raw[4] == 0x32 || raw[4] == 0x33
}
// Pyc matches a Python compiled file.
// The signatures are sourced from libmagic v5.47
func Pyc(raw []byte, limit uint32) bool {
if len(raw) < 8 {
return false
}
// python 1.0 through 3.7 signatures, magic/Magdir/python:13:190
pycMagic := []uint32{
0x02099900, 0x03099900, 0x892e0d0a, 0x04170d0a, 0x994e0d0a, 0xfcc40d0a,
0xfdc40d0a, 0x87c60d0a, 0x88c60d0a, 0x2aeb0d0a, 0x2beb0d0a, 0x2ded0d0a,
0x2eed0d0a, 0x3bf20d0a, 0x3cf20d0a, 0x45f20d0a, 0x59f20d0a, 0x63f20d0a,
0x6df20d0a, 0x6ef20d0a, 0x77f20d0a, 0x81f20d0a, 0x8bf20d0a, 0x8cf20d0a,
0x95f20d0a, 0x9ff20d0a, 0xa9f20d0a, 0xb3f20d0a, 0xb4f20d0a, 0xc7f20d0a,
0xd1f20d0a, 0xd2f20d0a, 0xdbf20d0a, 0xe5f20d0a, 0xeff20d0a, 0xf9f20d0a,
0x03f30d0a, 0x04f30d0a, 0x0af30d0a, 0xb80b0d0a, 0xc20b0d0a, 0xcc0b0d0a,
0xd60b0d0a, 0xe00b0d0a, 0xea0b0d0a, 0xf40b0d0a, 0xf50b0d0a, 0xff0b0d0a,
0x090c0d0a, 0x130c0d0a, 0x1d0c0d0a, 0x1f0c0d0a, 0x270c0d0a, 0x3b0c0d0a,
0x450c0d0a, 0x4f0c0d0a, 0x580c0d0a, 0x620c0d0a, 0x6c0c0d0a, 0x760c0d0a,
0x800c0d0a, 0x8a0c0d0a, 0x940c0d0a, 0x9e0c0d0a, 0xb20c0d0a, 0xbc0c0d0a,
0xc60c0d0a, 0xd00c0d0a, 0xda0c0d0a, 0xe40c0d0a, 0xee0c0d0a, 0xf80c0d0a,
0x020d0d0a, 0x0c0d0d0a, 0x160d0d0a, 0x170d0d0a, 0x200d0d0a, 0x210d0d0a,
0x2a0d0d0a, 0x2b0d0d0a, 0x2c0d0d0a, 0x2d0d0d0a, 0x2f0d0d0a, 0x300d0d0a,
0x310d0d0a, 0x320d0d0a, 0x330d0d0a, 0x3e0d0d0a, 0x3f0d0d0a,
}
n := binary.BigEndian.Uint32(raw)
if slices.Contains(pycMagic, n) {
return true
}
if raw[2] == 0x0d && raw[3] == 0x0a {
// Only two bits of flag field are currently used.
if l := binary.LittleEndian.Uint32(raw[4:]); l > 3 {
return false
}
if raw[1] == 0x0d || raw[1] == 0x0e {
return true
}
// PyPy magic numbers, magic/Magdir/python:233
n := binary.LittleEndian.Uint16(raw)
return n == 240 || n == 256 || n == 336 || n == 384 || n == 416
}
return false
}
// Pcap identifies "libpcap" capture files.
// https://www.tcpdump.org/manpages/pcap-savefile.5.html
func Pcap(raw []byte, _ uint32) bool {
if len(raw) < 4 {
return false
}
be := binary.BigEndian.Uint32(raw)
le := binary.LittleEndian.Uint32(raw)
return be == 0xa1b2c3d4 || be == 0xa1b23c4d ||
le == 0xa1b2c3d4 || le == 0xa1b23c4d
}

View File

@@ -3,6 +3,7 @@ package magic
import (
"bytes"
"encoding/binary"
"slices"
)
// Woff matches a Web Open Font Format file.
@@ -29,12 +30,13 @@ func Ttf(raw []byte, limit uint32) bool {
if !bytes.HasPrefix(raw, []byte{0x00, 0x01, 0x00, 0x00}) {
return false
}
// We cannot rely on the first 4 bytes because of false-positives.
// We have to digg deeper into the SFNT tables.
return hasSFNTTable(raw)
}
func hasSFNTTable(raw []byte) bool {
// 49 possible tables as explained below
if len(raw) < 16 || binary.BigEndian.Uint16(raw[4:]) >= 49 {
if len(raw) < 16 {
return false
}
@@ -87,14 +89,45 @@ func hasSFNTTable(raw []byte) bool {
0x6e616d65, // "name"
0x6f706264, // "opbd"
0x4f532f32, // "OS/2"
// The above tables come from the original Apple TTF specification,
// but the later Microsoft specification has additional tables.
// Common tables: https://learn.microsoft.com/en-us/typography/opentype/spec/otvarcommonformats
// Layout tables: https://learn.microsoft.com/en-us/typography/opentype/spec/chapter2
// Even if the Microsoft specification says OpenType, the tables are
// valid for TrueType as well.
0x47535542, // "GSUB"
0x47504f53, // "GPOS"
0x42415345, // "BASE"
0x4a535446, // "JSTF"
0x47444546, // "GDEF"
0x4d415448, // "MATH"
0x43424454, // "CBDT"
0x43424c43, // "CBLC"
0x43464620, // "CFF "
0x43464632, // "CFF2"
0x434f4c52, // "COLR"
0x4350414c, // "CPAL"
0x44534947, // "DSIG"
0x45424454, // "EBDT"
0x45424c43, // "EBLC"
0x48564152, // "HVAR"
0x4c545348, // "LTSH"
0x4d455247, // "MERG"
0x4d564152, // "MVAR"
0x50434c54, // "PCLT"
0x706f7374, // "post"
0x70726570, // "prep"
0x73626978, // "sbix"
0x53544154, // "STAT"
0x53564720, // "SVG "
0x56444d58, // "VDMX"
0x76686561, // "vhea"
0x766d7478, // "vmtx"
0x564f5247, // "VORG"
0x56564152, // "VVAR"
}
ourTable := binary.BigEndian.Uint32(raw[12:16])
for _, t := range possibleTables {
if ourTable == t {
return true
}
}
return false
return slices.Contains(possibleTables, ourTable)
}
// Eot matches an Embedded OpenType font file.

View File

@@ -3,6 +3,7 @@ package magic
import (
"bytes"
"encoding/binary"
"slices"
)
// Shp matches a shape format file.
@@ -39,13 +40,7 @@ func Shp(raw []byte, limit uint32) bool {
31, // MultiPatch
}
for _, st := range shapeTypes {
if st == int(binary.LittleEndian.Uint32(raw[108:112])) {
return true
}
}
return false
return slices.Contains(shapeTypes, int(binary.LittleEndian.Uint32(raw[108:112])))
}
// Shx matches a shape index format file.

View File

@@ -4,6 +4,8 @@ import (
"bytes"
"encoding/binary"
"slices"
"github.com/gabriel-vasile/mimetype/internal/scan"
)
// Png matches a Portable Network Graphics file.
@@ -15,7 +17,31 @@ func Png(raw []byte, _ uint32) bool {
// Apng matches an Animated Portable Network Graphics file.
// https://wiki.mozilla.org/APNG_Specification
func Apng(raw []byte, _ uint32) bool {
return offset(raw, []byte("acTL"), 37)
b := scan.Bytes(raw)
b.Advance(8) // the first 8 bytes matched by regular png
// PNG chunks are composed of:
// 4 bytes: length in big endian
// 4 bytes: chunk type
// length bytes: chunk data
// 4 bytes: CRC
//
// Limit to 32, so we don't waste time on huge inputs.
// acTL chunk must come before any IDAT chunks.
// https://www.w3.org/TR/png-3/#structure
for i := 0; i < 32 && len(b) > 0; i++ {
sz, _ := b.Uint32be()
if bytes.HasPrefix(b, []byte("acTL")) {
return true
}
if bytes.HasPrefix(b, []byte("IDAT")) {
return false
}
if !b.Advance(int(sz + 8)) {
return false
}
}
return false
}
// Jpg matches a Joint Photographic Experts Group file.

View File

@@ -136,6 +136,11 @@ func ftyp(raw []byte, sigs ...[]byte) bool {
return false
}
type shebangSig struct {
sig []byte
flag scan.Flags
}
// A valid shebang starts with the "#!" characters,
// followed by any number of spaces,
// followed by the path to the interpreter,
@@ -146,7 +151,7 @@ func ftyp(raw []byte, sigs ...[]byte) bool {
// #! /usr/bin/env php
//
// /usr/bin/env is the interpreter, php is the first and only argument.
func shebang(b scan.Bytes, matchFlags scan.Flags, sigs ...[]byte) bool {
func shebang(b scan.Bytes, sigs ...shebangSig) bool {
line := b.Line()
if len(line) < 2 || line[0] != '#' || line[1] != '!' {
return false
@@ -154,7 +159,7 @@ func shebang(b scan.Bytes, matchFlags scan.Flags, sigs ...[]byte) bool {
line = line[2:]
line.TrimLWS()
for _, s := range sigs {
if line.Match(s, matchFlags) != -1 {
if line.Match(s.sig, s.flag) != -1 {
return true
}
}

View File

@@ -3,6 +3,8 @@ package magic
import (
"bytes"
"encoding/binary"
"github.com/gabriel-vasile/mimetype/internal/cdf"
)
// Xlsx matches a Microsoft Excel 2007 file.
@@ -47,6 +49,15 @@ func Ole(raw []byte, limit uint32) bool {
// Doc matches a Microsoft Word 97-2003 file.
// See: https://github.com/decalage2/oletools/blob/412ee36ae45e70f42123e835871bac956d958461/oletools/common/clsid.py
func Doc(raw []byte, _ uint32) bool {
fromParsing := cdf.Detect(raw)
if fromParsing == cdf.CDFTypeDoc {
return true
}
if fromParsing != cdf.CDFTypeGeneric {
return false
}
// Fallback for inputs where the CDF directory is past the read limit: match
// the root storage CLSID, which often lies within the first sectors.
clsids := [][]byte{
// Microsoft Word 97-2003 Document (Word.Document.8)
{0x06, 0x09, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46},
@@ -55,19 +66,25 @@ func Doc(raw []byte, _ uint32) bool {
// Microsoft Word Picture (Word.Picture.8)
{0x07, 0x09, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46},
}
for _, clsid := range clsids {
if matchOleClsid(raw, clsid) {
return true
}
}
return false
}
// Ppt matches a Microsoft PowerPoint 97-2003 file or a PowerPoint 95 presentation.
func Ppt(raw []byte, limit uint32) bool {
// Root CLSID test is the safest way to detect identify OLE, however, the format
fromParsing := cdf.Detect(raw)
if fromParsing == cdf.CDFTypePpt {
return true
}
if fromParsing != cdf.CDFTypeGeneric {
return false
}
// Fallback for inputs where the CDF directory is past the read limit.
// Root CLSID test is the safest way to identify the OLE, however, the format
// often places the root CLSID at the end of the file.
if matchOleClsid(raw, []byte{
0x10, 0x8d, 0x81, 0x64, 0x9b, 0x4f, 0xcf, 0x11,
@@ -94,18 +111,21 @@ func Ppt(raw []byte, limit uint32) bool {
}
}
if bytes.HasPrefix(raw[512:], []byte{0xFD, 0xFF, 0xFF, 0xFF}) &&
raw[518] == 0x00 && raw[519] == 0x00 {
return true
}
return lin > 1152 && bytes.Contains(raw[1152:min(4096, lin)],
[]byte("P\x00o\x00w\x00e\x00r\x00P\x00o\x00i\x00n\x00t\x00 D\x00o\x00c\x00u\x00m\x00e\x00n\x00t"))
}
// Xls matches a Microsoft Excel 97-2003 file.
func Xls(raw []byte, limit uint32) bool {
// Root CLSID test is the safest way to detect identify OLE, however, the format
fromParsing := cdf.Detect(raw)
if fromParsing == cdf.CDFTypeXls {
return true
}
if fromParsing != cdf.CDFTypeGeneric {
return false
}
// Fallback for inputs where the CDF directory is past the read limit.
// Root CLSID test is the safest way to identify the OLE, however, the format
// often places the root CLSID at the end of the file.
if matchOleClsid(raw, []byte{
0x10, 0x08, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -148,6 +168,15 @@ func Pub(raw []byte, limit uint32) bool {
// Msg matches a Microsoft Outlook email file.
func Msg(raw []byte, limit uint32) bool {
fromParsing := cdf.Detect(raw)
if fromParsing == cdf.CDFTypeMsg {
return true
}
if fromParsing != cdf.CDFTypeGeneric {
return false
}
// Fallback for inputs where the CDF directory does not carry the streams the
// parser keys on: match the root storage CLSID instead.
return matchOleClsid(raw, []byte{
0x0B, 0x0D, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,
0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46,
@@ -157,10 +186,7 @@ func Msg(raw []byte, limit uint32) bool {
// Msi matches a Microsoft Windows Installer file.
// http://fileformats.archiveteam.org/wiki/Microsoft_Compound_File
func Msi(raw []byte, limit uint32) bool {
return matchOleClsid(raw, []byte{
0x84, 0x10, 0x0C, 0x00, 0x00, 0x00, 0x00, 0x00,
0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46,
})
return cdf.Detect(raw) == cdf.CDFTypeInstaller
}
// One matches a Microsoft OneNote file.

View File

@@ -130,6 +130,14 @@ func Xfdf(raw []byte, _ uint32) bool {
return xml(raw, xmlSig{[]byte("<xfdf"), []byte(`xmlns="http://ns.adobe.com/xfdf/"`)})
}
// CDXXML matches a CycloneDX XML BOM file.
// https://cyclonedx.org/docs/1.7/xml/
func CDXXML(raw []byte, _ uint32) bool {
// xmlns is missing the version suffix because there are too many past versions
// and probably future versions to come.
return xml(raw, xmlSig{[]byte("<bom"), []byte(`xmlns="http://cyclonedx.org/schema/bom/`)})
}
// VCard matches a Virtual Contact File.
func VCard(raw []byte, _ uint32) bool {
return ciPrefix(raw, []byte("BEGIN:VCARD\n"), []byte("BEGIN:VCARD\r\n"))
@@ -139,6 +147,14 @@ func VCard(raw []byte, _ uint32) bool {
func ICalendar(raw []byte, _ uint32) bool {
return ciPrefix(raw, []byte("BEGIN:VCALENDAR\n"), []byte("BEGIN:VCALENDAR\r\n"))
}
const (
snone = 0
scws = scan.CompactWS
sfw = scan.FullWord
scwsfw = scan.CompactWS | scan.FullWord
)
func phpPageF(raw []byte, _ uint32) bool {
return ciPrefix(raw,
[]byte("<?PHP"),
@@ -149,66 +165,61 @@ func phpPageF(raw []byte, _ uint32) bool {
}
func phpScriptF(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS,
[]byte("/usr/local/bin/php"),
[]byte("/usr/bin/php"),
[]byte("/usr/bin/env php"),
[]byte("/usr/bin/env -S php"),
shebangSig{[]byte("/usr/local/bin/php"), snone},
shebangSig{[]byte("/usr/bin/php"), snone},
shebangSig{[]byte("/usr/bin/env php"), scws},
shebangSig{[]byte("/usr/bin/env -S php"), scws},
)
}
// Js matches a Javascript file.
func Js(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS,
[]byte("/bin/node"),
[]byte("/usr/bin/node"),
[]byte("/bin/nodejs"),
[]byte("/usr/bin/nodejs"),
[]byte("/usr/bin/env node"),
[]byte("/usr/bin/env -S node"),
[]byte("/usr/bin/env nodejs"),
[]byte("/usr/bin/env -S nodejs"),
shebangSig{[]byte("/bin/node"), snone},
shebangSig{[]byte("/usr/bin/node"), snone},
shebangSig{[]byte("/bin/nodejs"), snone},
shebangSig{[]byte("/usr/bin/nodejs"), snone},
shebangSig{[]byte("/usr/bin/env node"), scws},
shebangSig{[]byte("/usr/bin/env -S node"), scws},
shebangSig{[]byte("/usr/bin/env nodejs"), scws},
shebangSig{[]byte("/usr/bin/env -S nodejs"), scws},
)
}
// Lua matches a Lua programming language file.
func Lua(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS|scan.FullWord,
[]byte("/usr/bin/lua"),
[]byte("/usr/local/bin/lua"),
[]byte("/usr/bin/env lua"),
[]byte("/usr/bin/env -S lua"),
shebangSig{[]byte("/usr/bin/lua"), sfw},
shebangSig{[]byte("/usr/local/bin/lua"), sfw},
shebangSig{[]byte("/usr/bin/env lua"), scwsfw},
shebangSig{[]byte("/usr/bin/env -S lua"), scwsfw},
)
}
// Perl matches a Perl programming language file.
func Perl(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS|scan.FullWord,
[]byte("/usr/bin/perl"),
[]byte("/usr/bin/env perl"),
[]byte("/usr/bin/env -S perl"),
shebangSig{[]byte("/usr/bin/perl"), sfw},
shebangSig{[]byte("/usr/bin/env perl"), scwsfw},
shebangSig{[]byte("/usr/bin/env -S perl"), scwsfw},
)
}
// Python matches a Python programming language file.
func Python(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS,
[]byte("/usr/bin/python"),
[]byte("/usr/local/bin/python"),
[]byte("/usr/bin/env python"),
[]byte("/usr/bin/env -S python"),
[]byte("/usr/bin/python2"),
[]byte("/usr/local/bin/python2"),
[]byte("/usr/bin/env python2"),
[]byte("/usr/bin/env -S python2"),
[]byte("/usr/bin/python3"),
[]byte("/usr/local/bin/python3"),
[]byte("/usr/bin/env python3"),
[]byte("/usr/bin/env -S python3"),
shebangSig{[]byte("/usr/bin/python"), snone},
shebangSig{[]byte("/usr/local/bin/python"), snone},
shebangSig{[]byte("/usr/bin/env python"), scws},
shebangSig{[]byte("/usr/bin/env -S python"), scws},
shebangSig{[]byte("/usr/bin/python2"), snone},
shebangSig{[]byte("/usr/local/bin/python2"), snone},
shebangSig{[]byte("/usr/bin/env python2"), scws},
shebangSig{[]byte("/usr/bin/env -S python2"), scws},
shebangSig{[]byte("/usr/bin/python3"), snone},
shebangSig{[]byte("/usr/local/bin/python3"), snone},
shebangSig{[]byte("/usr/bin/env python3"), scws},
shebangSig{[]byte("/usr/bin/env -S python3"), scws},
)
}
@@ -216,30 +227,28 @@ func Python(raw []byte, _ uint32) bool {
// Ruby matches a Ruby programming language file.
func Ruby(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS,
[]byte("/usr/bin/ruby"),
[]byte("/usr/local/bin/ruby"),
[]byte("/usr/bin/env ruby"),
[]byte("/usr/bin/env -S ruby"),
shebangSig{[]byte("/usr/bin/ruby"), snone},
shebangSig{[]byte("/usr/local/bin/ruby"), snone},
shebangSig{[]byte("/usr/bin/env ruby"), scws},
shebangSig{[]byte("/usr/bin/env -S ruby"), scws},
)
}
// Tcl matches a Tcl programming language file.
func Tcl(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS,
[]byte("/usr/bin/tcl"),
[]byte("/usr/local/bin/tcl"),
[]byte("/usr/bin/env tcl"),
[]byte("/usr/bin/env -S tcl"),
[]byte("/usr/bin/tclsh"),
[]byte("/usr/local/bin/tclsh"),
[]byte("/usr/bin/env tclsh"),
[]byte("/usr/bin/env -S tclsh"),
[]byte("/usr/bin/wish"),
[]byte("/usr/local/bin/wish"),
[]byte("/usr/bin/env wish"),
[]byte("/usr/bin/env -S wish"),
shebangSig{[]byte("/usr/bin/tcl"), snone},
shebangSig{[]byte("/usr/local/bin/tcl"), snone},
shebangSig{[]byte("/usr/bin/env tcl"), scws},
shebangSig{[]byte("/usr/bin/env -S tcl"), scws},
shebangSig{[]byte("/usr/bin/tclsh"), snone},
shebangSig{[]byte("/usr/local/bin/tclsh"), snone},
shebangSig{[]byte("/usr/bin/env tclsh"), scws},
shebangSig{[]byte("/usr/bin/env -S tclsh"), scws},
shebangSig{[]byte("/usr/bin/wish"), snone},
shebangSig{[]byte("/usr/local/bin/wish"), snone},
shebangSig{[]byte("/usr/bin/env wish"), scws},
shebangSig{[]byte("/usr/bin/env -S wish"), scws},
)
}
@@ -251,32 +260,31 @@ func Rtf(raw []byte, _ uint32) bool {
// Shell matches a shell script file.
func Shell(raw []byte, _ uint32) bool {
return shebang(raw,
scan.CompactWS|scan.FullWord,
[]byte("/bin/sh"),
[]byte("/bin/bash"),
[]byte("/usr/local/bin/bash"),
[]byte("/usr/bin/env bash"),
[]byte("/usr/bin/env -S bash"),
[]byte("/bin/csh"),
[]byte("/usr/local/bin/csh"),
[]byte("/usr/bin/env csh"),
[]byte("/usr/bin/env -S csh"),
[]byte("/bin/dash"),
[]byte("/usr/local/bin/dash"),
[]byte("/usr/bin/env dash"),
[]byte("/usr/bin/env -S dash"),
[]byte("/bin/ksh"),
[]byte("/usr/local/bin/ksh"),
[]byte("/usr/bin/env ksh"),
[]byte("/usr/bin/env -S ksh"),
[]byte("/bin/tcsh"),
[]byte("/usr/local/bin/tcsh"),
[]byte("/usr/bin/env tcsh"),
[]byte("/usr/bin/env -S tcsh"),
[]byte("/bin/zsh"),
[]byte("/usr/local/bin/zsh"),
[]byte("/usr/bin/env zsh"),
[]byte("/usr/bin/env -S zsh"),
shebangSig{[]byte("/bin/sh"), sfw},
shebangSig{[]byte("/bin/bash"), sfw},
shebangSig{[]byte("/usr/local/bin/bash"), sfw},
shebangSig{[]byte("/usr/bin/env bash"), scwsfw},
shebangSig{[]byte("/usr/bin/env -S bash"), scwsfw},
shebangSig{[]byte("/bin/csh"), sfw},
shebangSig{[]byte("/usr/local/bin/csh"), sfw},
shebangSig{[]byte("/usr/bin/env csh"), scwsfw},
shebangSig{[]byte("/usr/bin/env -S csh"), scwsfw},
shebangSig{[]byte("/bin/dash"), sfw},
shebangSig{[]byte("/usr/local/bin/dash"), sfw},
shebangSig{[]byte("/usr/bin/env dash"), scwsfw},
shebangSig{[]byte("/usr/bin/env -S dash"), scwsfw},
shebangSig{[]byte("/bin/ksh"), sfw},
shebangSig{[]byte("/usr/local/bin/ksh"), sfw},
shebangSig{[]byte("/usr/bin/env ksh"), scwsfw},
shebangSig{[]byte("/usr/bin/env -S ksh"), scwsfw},
shebangSig{[]byte("/bin/tcsh"), sfw},
shebangSig{[]byte("/usr/local/bin/tcsh"), sfw},
shebangSig{[]byte("/usr/bin/env tcsh"), scwsfw},
shebangSig{[]byte("/usr/bin/env -S tcsh"), scwsfw},
shebangSig{[]byte("/bin/zsh"), sfw},
shebangSig{[]byte("/usr/local/bin/zsh"), sfw},
shebangSig{[]byte("/usr/bin/env zsh"), scwsfw},
shebangSig{[]byte("/usr/bin/env -S zsh"), scwsfw},
)
}
@@ -352,6 +360,12 @@ func GLTF(raw []byte, limit uint32) bool {
return jsonHelper(raw, limit, json.QueryGLTF, json.TokObject)
}
// CDXJSON matches a CycloneDX JSON BOM file.
// https://cyclonedx.org/docs/1.7/json/
func CDXJSON(raw []byte, limit uint32) bool {
return jsonHelper(raw, limit, json.QueryCDX, json.TokObject)
}
// jsonHelper parses raw and tries to match the q query against it. wantToks
// ensures we're not wasting time parsing an input that would not pass anyway,
// ex: the input is a valid JSON array, but we're looking for a JSON object.
@@ -393,8 +407,16 @@ func NdJSON(raw []byte, limit uint32) bool {
var l scan.Bytes
for len(s) != 0 {
l = s.Line()
_, inspected, firstToken, _ := json.Parse(json.QueryNone, l)
if len(l) != inspected {
parsed, inspected, firstToken, _ := json.Parse(json.QueryNone, l)
// Only the last line may be truncated by the read limit; for it, it is
// enough that the parser inspected all of it. Every other line must be a
// complete, valid JSON document, otherwise a single JSON document spread
// over multiple lines would be mistaken for NDJSON. #803
if len(s) == 0 {
if inspected != len(l) {
return false
}
} else if parsed != len(l) {
return false
}
if firstToken == json.TokArray || firstToken == json.TokObject {
@@ -563,6 +585,8 @@ func RFC822(raw []byte, limit uint32) bool {
// Some of the hints are IgnoreCase, some not. I selected based on what libmagic
// does and based on personal observations from sample files.
hints := []rfc822Hint{
// Enron dataset has Message-ID, Message-Id and Message-id.
{[]byte("Message-ID: "), scan.IgnoreCase},
{[]byte("From: "), 0},
{[]byte("To: "), 0},
{[]byte("CC: "), scan.IgnoreCase},
@@ -599,3 +623,44 @@ func lineHasRFC822Hint(b scan.Bytes, hints []rfc822Hint) bool {
}
return false
}
func GEDCOM(raw []byte, limit uint32) bool {
// Skip if empty
if len(raw) == 0 {
return false
}
// GEDCOM header fits within first 4KB
searchLimit := min(len(raw), 4096)
raw = raw[:searchLimit]
b := scan.Bytes(raw)
// Skip BOM if present: UTF-8, UTF-16BE, UTF-16LE
for _, bom := range [][]byte{
{0xEF, 0xBB, 0xBF}, // UTF-8
{0xFE, 0xFF}, // UTF-16BE
{0xFF, 0xFE}, // UTF-16LE
} {
if bytes.HasPrefix(b, bom) {
b.Advance(len(bom))
break // Only one BOM can exist at the start
}
}
b.TrimLWS()
firstLine := b.Line()
if !bytes.Equal(firstLine, []byte("0 HEAD")) {
return false
}
// "1 GEDC" is mandatory in the header
for i := 0; i < 10; i++ {
line := b.Line()
if bytes.Equal(line, []byte("1 GEDC")) {
return true
}
}
return false
}

View File

@@ -17,8 +17,7 @@ func TSV(raw []byte, limit uint32) bool {
func sv(in []byte, comma byte, limit uint32) bool {
s := scan.Bytes(in)
s.DropLastLine(limit)
r := csv.NewParser(comma, '#', s)
r := csv.NewParser(comma, '#', &s)
headerFields, _, hasMore := r.CountFields(false)
if headerFields < 2 || !hasMore {
@@ -30,8 +29,22 @@ func sv(in []byte, comma byte, limit uint32) bool {
if !hasMore && fields == 0 {
break
}
csvLines++
if fields != headerFields {
if fields == headerFields {
csvLines++
} else {
// maybeTruncated signals the input was cut at the read limit,
// meaning the last line may be an incomplete CSV record.
maybeTruncated := limit > 0 && uint64(len(in)) >= uint64(limit)
if maybeTruncated && fields < headerFields {
// Allow the last row to have any number of fields
// if the input is maybeTruncated.
// BUG: if len(input) == limit, then the input is not truncated
// but it is still allowed to have the wrong number of fields
// and it will be reported as valid CSV.
if len(s) == 0 {
break
}
}
return false
}
if csvLines >= 10 {

View File

@@ -49,10 +49,8 @@ func isMatroskaFileTypeMatched(in []byte, flType string) bool {
// The logic of search is: find first instance of \x42\x82 and then
// search for given string after n bytes of above instance.
func isFileTypeNamePresent(in []byte, flType string) bool {
ind, maxInd, lenIn := 0, 4096, len(in)
if lenIn < maxInd { // restricting length to 4096
maxInd = lenIn
}
ind, lenIn := 0, len(in)
maxInd := min(4096, lenIn)
ind = bytes.Index(in[:maxInd], []byte("\x42\x82"))
if ind > 0 && lenIn > ind+2 {
ind += 2

View File

@@ -187,11 +187,11 @@ func msoxml(raw scan.Bytes, searchFor zipEntries, stopAfter int) bool {
return false
}
var zipLocalFileHeader = []byte("PK\003\004")
// next extracts the name of the next zip entry.
func (i *zipIterator) next() []byte {
pk := []byte("PK\003\004")
n := bytes.Index(i.b, pk)
n := bytes.Index(i.b, zipLocalFileHeader)
if n == -1 {
return nil
}
@@ -212,10 +212,85 @@ func (i *zipIterator) next() []byte {
return i.b[:l]
}
// skipZipflingerEntry tries to detect a Zipflinger virtual entry and skips it.
// The detection is based on the following properties:
// - compression method is 0
// - CRC32 is 0
// - compressed size is 0
// - uncompressed size is 0
// - file name is empty
// Returns true if it was found and skipped.
func (i *zipIterator) skipZipflingerEntry() (skipped bool) {
// Make a backup of the data so the inspection does not loses it.
b := i.b
defer func() {
// If no zipflinger was found, restore the original data.
if !skipped {
i.b = b
}
}()
n := bytes.Index(i.b, zipLocalFileHeader)
if n == -1 {
return false
}
if !i.b.Advance(0x08) {
return false
}
// Check compression method
if cm, ok := i.b.Uint16(); !ok || cm != 0 {
return false
}
// Advance up to the CRC32 field
if !i.b.Advance(0x04) {
return false
}
// Check CRC32
if crc32, ok := i.b.Uint32(); !ok || crc32 != 0 {
return false
}
// Check compressed size
if compressedSize, ok := i.b.Uint32(); !ok || compressedSize != 0 {
return false
}
// Check uncompressed size
if uncompressedSize, ok := i.b.Uint32(); !ok || uncompressedSize != 0 {
return false
}
// Check for empty file name
if l, ok := i.b.Uint16(); !ok || l != 0 {
return false
}
// Reached a zipflinger virtual entry: skip extra data
l, ok := i.b.Uint16()
if !ok {
return false
}
if !i.b.Advance(int(l)) {
return false
}
return true
}
// APK matches an Android Package Archive.
// The source of signatures is https://github.com/file/file/blob/1778642b8ba3d947a779a36fcd81f8e807220a19/magic/Magdir/archive#L1820-L1887
func APK(raw []byte, _ uint32) bool {
return zipHas(raw, zipEntries{{
iter := zipIterator{raw}
// If a Zipflinger Virtual Entry is detected, then the data is considered APK
if iter.skipZipflingerEntry() {
return true
}
return zipHas(iter.b, zipEntries{{
name: []byte("AndroidManifest.xml"),
}, {
name: []byte("META-INF/com/android/build/gradle/app-metadata.properties"),

View File

@@ -0,0 +1,140 @@
package mp3
import "bytes"
// minTruncatedSyncMatches is the minimum number of confirmed successive
// header matches required to accept a candidate frame when the buffer ends
// before maxFrameSyncMatches confirmations can be performed.
const minTruncatedSyncMatches = 2
func ExtractFrame(b []byte) (start, size int) {
limit := min(len(b), 2048+headerSize)
for i := 0; i < limit-headerSize; i++ {
j := bytes.IndexByte(b[i:limit-headerSize], 0xFF)
if j < 0 {
break
}
i += j
hdr := header{b[i], b[i+1], b[i+2], b[i+3]}
if !hdr.valid() {
continue
}
frameBytes := hdr.frameBytes()
frameAndPad := frameBytes + hdr.padding()
validHere := frameBytes > 0 && i+frameAndPad <= len(b) && matchFrame(b[i:])
// When the buffer is exactly one frame, matchFrame cannot look ahead for
// a subsequent header to confirm the stream. Trust the validated header.
exact := i == 0 && frameAndPad == len(b)
if validHere || exact {
return i, frameAndPad
}
}
return 0, 0
}
// matchFrame confirms a candidate header by stepping forward and checking that
// subsequent headers are consistent.
func matchFrame(buf []byte) bool {
// maxFrameSyncMatches limits how many valid frames we look at.
const maxFrameSyncMatches = 10
hdr := header{buf[0], buf[1], buf[2], buf[3]}
i := hdr.frameBytes() + hdr.padding()
for nmatch := 0; nmatch < maxFrameSyncMatches; nmatch++ {
if i+headerSize > len(buf) {
return nmatch >= minTruncatedSyncMatches
}
cmp := header{buf[i], buf[i+1], buf[i+2], buf[i+3]}
if !hdr.compatibleWith(cmp) {
return false
}
i += cmp.frameBytes() + cmp.padding()
}
return true
}
const headerSize = 4
type header [headerSize]byte
func (h header) isFreeFormat() bool { return h[2]&0xF0 == 0 }
func (h header) isMPEG1() bool { return h[1]&0x8 != 0 }
func (h header) isMPEG25() bool { return h[1]&0x10 == 0 }
func (h header) rawLayer() byte { return h[1] >> 1 & 3 }
func (h header) rawBitrate() byte { return h[2] >> 4 }
func (h header) rawSampleRate() byte { return h[2] >> 2 & 3 }
func (h header) rawEmphasis() byte { return h[3] & 0b11 }
func (h header) isFrame576() bool { return h[1]&14 == 2 }
func (h header) padding() int {
if h[2]&0x2 != 0 {
return 1
}
return 0
}
// valid reports whether the four bytes form a syntactically valid MP3 header.
func (h header) valid() bool {
return h[0] == 0xff &&
((h[1]&0xF0) == 0xf0 || (h[1]&0xFE) == 0xe2) &&
h.rawLayer() == 1 && // Layer III
h.rawBitrate() != 15 && // Not allowed by spec.
h.rawSampleRate() != 3 &&
h.rawEmphasis() != 2 &&
// The code for extracting frame size for free-format is tedious and
// free-format MP3s are extinct.
!h.isFreeFormat()
}
// compatibleWith reports whether two headers describe frames belonging to the
// same MP3 stream — same MPEG version, layer, sample-rate index.
func (h header) compatibleWith(o header) bool {
return o.valid() &&
(h[1]^o[1])&0xFE == 0 &&
(h[2]^o[2])&0x0C == 0
}
// bitrateKbps returns the bitrate of the frame in kilobits per second.
func (h header) bitrateKbps() int {
// halfrate[mpeg1?][bitrate_idx] holds bitrate/2 in kbps.
halfrate := [2][15]uint8{
{0, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 64, 72, 80},
{0, 16, 20, 24, 28, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160},
}
mpeg1 := 0
if h.isMPEG1() {
mpeg1 = 1
}
return 2 * int(halfrate[mpeg1][h.rawBitrate()])
}
// sampleRateHz returns the sampling rate of the frame in Hz.
func (h header) sampleRateHz() int {
base := [3]int{44100, 48000, 32000}[h.rawSampleRate()]
if !h.isMPEG1() {
base >>= 1
}
if h.isMPEG25() {
base >>= 1
}
return base
}
// frameSamples returns the number of audio samples per channel encoded in
// the frame.
func (h header) frameSamples() int {
if h.isFrame576() {
return 576
}
return 1152
}
// frameBytes returns the size of the frame body (header + side info + audio
// data, excluding padding) in bytes.
func (h header) frameBytes() int {
br := h.bitrateKbps()
sr := h.sampleRateHz()
if br == 0 || sr == 0 {
return 0
}
return h.frameSamples() * br * 125 / sr
}

View File

@@ -122,26 +122,6 @@ func (b *Bytes) Line() Bytes {
return line
}
// DropLastLine drops the last incomplete line from b.
//
// mimetype limits itself to ReadLimit bytes when performing a detection.
// This means, for file formats like CSV for NDJSON, the last line of the input
// can be an incomplete line.
// If b length is less than readLimit, it means we received an incomplete file
// and proceed with dropping the last line.
func (b *Bytes) DropLastLine(readLimit uint32) {
if readLimit == 0 || uint64(len(*b)) < uint64(readLimit) {
return
}
for i := len(*b) - 1; i > 0; i-- {
if (*b)[i] == '\n' {
*b = (*b)[:i]
return
}
}
}
func (b *Bytes) Uint16() (uint16, bool) {
if len(*b) < 2 {
return 0, false
@@ -151,6 +131,24 @@ func (b *Bytes) Uint16() (uint16, bool) {
return v, true
}
func (b *Bytes) Uint32() (uint32, bool) {
if len(*b) < 4 {
return 0, false
}
v := binary.LittleEndian.Uint32(*b)
*b = (*b)[4:]
return v, true
}
func (b *Bytes) Uint32be() (uint32, bool) {
if len(*b) < 4 {
return 0, false
}
v := binary.BigEndian.Uint32(*b)
*b = (*b)[4:]
return v, true
}
type Flags int
const (
@@ -205,10 +203,8 @@ func (b Bytes) Match(p []byte, flags Flags) int {
if l == 0 {
return -1
}
// If no flags, or scanning for full word at the end of pattern then
// do a fast HasPrefix check.
// For other flags it's not possible to use HasPrefix.
if flags == 0 || flags&FullWord > 0 {
// Some cases we can handle with a simple bytes.HasPrefix.
if flags == 0 || flags == FullWord {
if bytes.HasPrefix(b, p) {
b = b[len(p):]
p = p[len(p):]
@@ -232,7 +228,7 @@ func (b Bytes) Match(p []byte, flags Flags) int {
return -1
}
b = b[1:]
if !ByteIsWS(p[0]) {
if len(p) > 0 && !ByteIsWS(p[0]) {
b.TrimLWS()
}
} else {

View File

@@ -23,6 +23,14 @@ type MIME struct {
}
// String returns the string representation of the MIME type, e.g., "application/zip".
// String return values can change between releases, for example, when [IANA]
// assigns a new media type. Use [MIME.Is] to avoid breaking changes.
//
// mtype := mimetype.Detect(zipFile)
// if mtype.String() == "application/zip" { /* Plain string comparison is brittle. */ }
// if mtype.Is("application/zip") { /* Will continue to work between releases */ }
//
// [IANA]: https://www.iana.org/assignments/media-types/media-types.xhtml
func (m *MIME) String() string {
return m.mime
}
@@ -38,17 +46,19 @@ func (m *MIME) Extension() string {
// Each MIME type has a non-nil parent, except for the root MIME type.
//
// For example, the application/json and text/html MIME types have text/plain as
// their parent because they are text files who happen to contain JSON or HTML.
// their parent because they are text files that happen to contain JSON or HTML.
// Another example is the ZIP format, which is used as container
// for Microsoft Office files, EPUB files, JAR files, and others.
func (m *MIME) Parent() *MIME {
return m.parent
}
// Is checks whether this MIME type, or any of its aliases, is equal to the
// Is checks whether this MIME type, or any of its [aliases], is equal to the
// expected MIME type. MIME type equality test is done on the "type/subtype"
// section, ignores any optional MIME parameters, ignores any leading and
// trailing whitespace, and is case insensitive.
//
// [aliases]: https://github.com/gabriel-vasile/mimetype/blob/master/supported_mimes.md
func (m *MIME) Is(expectedMIME string) bool {
// Parsing is needed because some detected MIME types contain parameters
// that need to be stripped for the comparison.
@@ -129,7 +139,7 @@ func (m *MIME) flatten() []*MIME {
// hierarchy returns an easy to read list of ancestors for m.
// For example, application/json would return json>txt>root.
func (m *MIME) hierarchy() string {
h := ""
var h strings.Builder
for m := m; m != nil; m = m.Parent() {
e := strings.TrimPrefix(m.Extension(), ".")
if e == "" {
@@ -142,9 +152,9 @@ func (m *MIME) hierarchy() string {
e = "root"
}
}
h += ">" + e
h.WriteString(">" + e)
}
return strings.TrimPrefix(h, ">")
return strings.TrimPrefix(h.String(), ">")
}
// clone creates a new MIME with the provided optional MIME parameters.

View File

@@ -1,6 +1,6 @@
// Package mimetype uses magic number signatures to detect the MIME type of a file.
//
// File formats are stored in a hierarchy with application/octet-stream at its root.
// File formats are stored in a hierarchy with "application/octet-stream" at its root.
// For example, the hierarchy for HTML format is application/octet-stream ->
// text/plain -> text/html.
package mimetype
@@ -12,14 +12,14 @@ import (
"sync/atomic"
)
const defaultLimit uint32 = 3072
const defaultLimit uint32 = 4096
// readLimit is the maximum number of bytes from the input used when detecting.
var readLimit uint32 = defaultLimit
// Detect returns the MIME type found from the provided byte slice.
//
// The result is always a valid MIME type, with application/octet-stream
// The result is always a valid MIME type, with "application/octet-stream"
// returned when identification failed.
func Detect(in []byte) *MIME {
// Using atomic because readLimit can be written at the same time in other goroutine.
@@ -34,7 +34,7 @@ func Detect(in []byte) *MIME {
// DetectReader returns the MIME type of the provided reader.
//
// The result is always a valid MIME type, with application/octet-stream
// The result is always a valid MIME type, with "application/octet-stream"
// returned when identification failed with or without an error.
// Any error returned is related to the reading from the input reader.
//
@@ -72,7 +72,7 @@ func DetectReader(r io.Reader) (*MIME, error) {
// DetectFile returns the MIME type of the provided file.
//
// The result is always a valid MIME type, with application/octet-stream
// The result is always a valid MIME type, with "application/octet-stream"
// returned when identification failed with or without an error.
// Any error returned is related to the opening and reading from the input file.
func DetectFile(path string) (*MIME, error) {
@@ -112,7 +112,7 @@ func SetLimit(limit uint32) {
}
// Extend adds detection for other file formats.
// It is equivalent to calling Extend() on the root MIME type "application/octet-stream".
// It is equivalent to calling [MIME.Extend] on the root MIME type "application/octet-stream".
func Extend(detector func(raw []byte, limit uint32) bool, mime, extension string, aliases ...string) {
root.Extend(detector, mime, extension, aliases...)
}

View File

@@ -1,4 +1,4 @@
## 199 Supported MIME types
## 204 Supported MIME types
This file is automatically generated when running tests. Do not edit manually.
Extension | MIME type <br> Aliases | Hierarchy
@@ -42,7 +42,7 @@ Extension | MIME type <br> Aliases | Hierarchy
**.oga** | **audio/ogg** | oga>ogg>root
**.ogv** | **video/ogg** | ogv>ogg>root
**.png** | **image/png** | png>root
**.png** | **image/vnd.mozilla.apng** | png>png>root
**.apng** | **image/apng** <br> image/vnd.mozilla.apng | apng>png>root
**.jpg** | **image/jpeg** | jpg>root
**.jxl** | **image/jxl** | jxl>root
**.jp2** | **image/jp2** | jp2>root
@@ -67,7 +67,6 @@ Extension | MIME type <br> Aliases | Hierarchy
**.bmp** | **image/bmp** <br> image/x-bmp, image/x-ms-bmp | bmp>root
**.123** | **application/vnd.lotus-1-2-3** | 123>root
**.ico** | **image/x-icon** | ico>root
**.mp3** | **audio/mpeg** <br> audio/x-mpeg, audio/mp3 | mp3>root
**.flac** | **audio/flac** | flac>root
**.midi** | **audio/midi** <br> audio/mid, audio/sp-midi, audio/x-mid, audio/x-midi | midi>root
**.ape** | **audio/ape** | ape>root
@@ -95,7 +94,7 @@ Extension | MIME type <br> Aliases | Hierarchy
**.webm** | **video/webm** <br> audio/webm | webm>root
**.avi** | **video/x-msvideo** <br> video/avi, video/msvideo | avi>root
**.flv** | **video/x-flv** | flv>root
**.mkv** | **video/x-matroska** | mkv>root
**.mkv** | **video/matroska** <br> video/x-matroska | mkv>root
**.asf** | **video/x-ms-asf** <br> video/asf, video/x-ms-wmv | asf>root
**.aac** | **audio/aac** | aac>root
**.voc** | **audio/x-unknown** | voc>root
@@ -116,7 +115,7 @@ Extension | MIME type <br> Aliases | Hierarchy
**.shp** | **application/vnd.shp** | shp>shx>root
**.dbf** | **application/x-dbf** | dbf>root
**.dcm** | **application/dicom** | dcm>root
**.rar** | **application/x-rar-compressed** <br> application/x-rar | rar>root
**.rar** | **application/vnd.rar** <br> application/x-rar-compressed, application/x-rar | rar>root
**.djvu** | **image/vnd.djvu** | djvu>root
**.mobi** | **application/x-mobipocket-ebook** | mobi>root
**.lit** | **application/x-ms-reader** | lit>root
@@ -158,6 +157,9 @@ Extension | MIME type <br> Aliases | Hierarchy
**.hlp** | **application/x-os2-hlp** | hlp>root
**.fm** | **application/vnd.framemaker** | fm>root
**.bufr** | **application/bufr** | bufr>root
**.pyc** | **application/x-bytecode.python** | pyc>root
**.pcap** | **application/vnd.tcpdump.pcap** | pcap>root
**.mp3** | **audio/mpeg** <br> audio/x-mpeg, audio/mp3 | mp3>root
**.txt** | **text/plain** | txt>root
**.svg** | **image/svg+xml** | svg>txt>root
**.html** | **text/html** | html>txt>root
@@ -176,6 +178,7 @@ Extension | MIME type <br> Aliases | Hierarchy
**.xfdf** | **application/vnd.adobe.xfdf** | xfdf>xml>txt>root
**.owl** | **application/owl+xml** | owl>xml>txt>root
**.html** | **application/xhtml+xml** | html>xml>txt>root
**.xml** | **application/vnd.cyclonedx+xml** | xml>xml>txt>root
**.php** | **text/x-php** | php>txt>root
**.js** | **text/javascript** <br> application/x-javascript, application/javascript | js>txt>root
**.lua** | **text/x-lua** | lua>txt>root
@@ -186,6 +189,7 @@ Extension | MIME type <br> Aliases | Hierarchy
**.geojson** | **application/geo+json** | geojson>json>txt>root
**.har** | **application/json** | har>json>txt>root
**.gltf** | **model/gltf+json** | gltf>json>txt>root
**.json** | **application/vnd.cyclonedx+json** | json>json>txt>root
**.ndjson** | **application/x-ndjson** | ndjson>txt>root
**.rtf** | **text/rtf** <br> application/rtf | rtf>txt>root
**.srt** | **application/x-subrip** <br> application/x-srt, text/x-srt | srt>txt>root
@@ -202,3 +206,4 @@ Extension | MIME type <br> Aliases | Hierarchy
**.ppm** | **image/x-portable-pixmap** | ppm>txt>root
**.pam** | **image/x-portable-arbitrarymap** | pam>txt>root
**.eml** | **message/rfc822** | eml>txt>root
**.ged** | **text/vnd.familysearch.gedcom** | ged>txt>root

View File

@@ -19,12 +19,16 @@ var root = newMIME("application/octet-stream", "",
func([]byte, uint32) bool { return true },
xpm, sevenZ, zip, pdf, fdf, ole, ps, psd, p7s, ogg, png, jpg, jxl, jp2, jpx,
jpm, jxs, gif, webp, exe, elf, ar, tar, xar, bz2, fits, tiff, bmp, lotus, ico,
mp3, flac, midi, ape, musePack, amr, wav, aiff, au, mpeg, quickTime, mp4, webM,
flac, midi, ape, musePack, amr, wav, aiff, au, mpeg, quickTime, mp4, webM,
avi, flv, mkv, asf, aac, voc, m3u, rmvb, gzip, class, swf, crx, ttf, woff,
woff2, otf, ttc, eot, wasm, shx, dbf, dcm, rar, djvu, mobi, lit, bpg, cbor,
sqlite3, dwg, nes, lnk, macho, qcp, icns, hdr, mrc, mdb, accdb, zstd, cab,
rpm, xz, lzip, torrent, cpio, tzif, xcf, pat, gbr, glb, cabIS, jxr, parquet,
oneNote, chm, wpd, dxf, grib, zlib, inf, hlp, fm, bufr,
oneNote, chm, wpd, dxf, grib, zlib, inf, hlp, fm, bufr, pyc, pcap,
// MP3 is late because it does a linear search in the input. That means
// containers that embed an MP3, for example: an mp4 file, or a zip without
// compression, would pass as MP3s.
mp3,
// Keep text last because it is the slowest check.
text,
)
@@ -82,16 +86,17 @@ var (
alias("application/x-ogg")
oggAudio = newMIME("audio/ogg", ".oga", magic.OggAudio)
oggVideo = newMIME("video/ogg", ".ogv", magic.OggVideo)
text = newMIME("text/plain", ".txt", magic.Text, svg, html, xml, php, js, lua, perl, python, ruby, json, ndJSON, rtf, srt, tcl, csv, tsv, vCard, iCalendar, warc, vtt, shell, netpbm, netpgm, netppm, netpam, rfc822)
xml = newMIME("text/xml", ".xml", magic.XML, rss, atom, x3d, kml, xliff, collada, gml, gpx, tcx, amf, threemf, xfdf, owl2, xhtml).
text = newMIME("text/plain", ".txt", magic.Text, svg, html, xml, php, js, lua, perl, python, ruby, json, ndJSON, rtf, srt, tcl, csv, tsv, vCard, iCalendar, warc, vtt, shell, netpbm, netpgm, netppm, netpam, rfc822, gedcom)
xml = newMIME("text/xml", ".xml", magic.XML, rss, atom, x3d, kml, xliff, collada, gml, gpx, tcx, amf, threemf, xfdf, owl2, xhtml, cdxxml).
alias("application/xml")
xhtml = newMIME("application/xhtml+xml", ".html", magic.XHTML)
json = newMIME("application/json", ".json", magic.JSON, geoJSON, har, gltf)
json = newMIME("application/json", ".json", magic.JSON, geoJSON, har, gltf, cdxJSON)
har = newMIME("application/json", ".har", magic.HAR)
csv = newMIME("text/csv", ".csv", magic.CSV)
tsv = newMIME("text/tab-separated-values", ".tsv", magic.TSV)
geoJSON = newMIME("application/geo+json", ".geojson", magic.GeoJSON)
ndJSON = newMIME("application/x-ndjson", ".ndjson", magic.NdJSON)
cdxJSON = newMIME("application/vnd.cyclonedx+json", ".json", magic.CDXJSON)
html = newMIME("text/html", ".html", magic.HTML)
php = newMIME("text/x-php", ".php", magic.Php)
rtf = newMIME("text/rtf", ".rtf", magic.Rtf).alias("application/rtf")
@@ -104,6 +109,7 @@ var (
perl = newMIME("text/x-perl", ".pl", magic.Perl)
python = newMIME("text/x-python", ".py", magic.Python).
alias("text/x-script.python", "application/x-python")
pyc = newMIME("application/x-bytecode.python", ".pyc", magic.Pyc)
ruby = newMIME("text/x-ruby", ".rb", magic.Ruby).
alias("application/x-ruby")
shell = newMIME("text/x-shellscript", ".sh", magic.Shell).
@@ -127,13 +133,15 @@ var (
tcx = newMIME("application/vnd.garmin.tcx+xml", ".tcx", magic.Tcx)
amf = newMIME("application/x-amf", ".amf", magic.Amf)
threemf = newMIME("application/vnd.ms-package.3dmanufacturing-3dmodel+xml", ".3mf", magic.Threemf)
cdxxml = newMIME("application/vnd.cyclonedx+xml", ".xml", magic.CDXXML)
png = newMIME("image/png", ".png", magic.Png, apng)
apng = newMIME("image/vnd.mozilla.apng", ".png", magic.Apng)
jpg = newMIME("image/jpeg", ".jpg", magic.Jpg)
jxl = newMIME("image/jxl", ".jxl", magic.Jxl)
jp2 = newMIME("image/jp2", ".jp2", magic.Jp2)
jpx = newMIME("image/jpx", ".jpf", magic.Jpx)
jpm = newMIME("image/jpm", ".jpm", magic.Jpm).
apng = newMIME("image/apng", ".apng", magic.Apng).
alias("image/vnd.mozilla.apng")
jpg = newMIME("image/jpeg", ".jpg", magic.Jpg)
jxl = newMIME("image/jxl", ".jxl", magic.Jxl)
jp2 = newMIME("image/jp2", ".jp2", magic.Jp2)
jpx = newMIME("image/jpx", ".jpf", magic.Jpx)
jpm = newMIME("image/jpm", ".jpm", magic.Jpm).
alias("video/jpm")
jxs = newMIME("image/jxs", ".jxs", magic.Jxs)
xpm = newMIME("image/x-xpixmap", ".xpm", magic.Xpm)
@@ -156,7 +164,7 @@ var (
heifSeq = newMIME("image/heif-sequence", ".heif", magic.HeifSequence)
hdr = newMIME("image/vnd.radiance", ".hdr", magic.Hdr)
avif = newMIME("image/avif", ".avif", magic.AVIF)
mp3 = newMIME("audio/mpeg", ".mp3", magic.Mp3).
mp3 = newMIME("audio/mpeg", ".mp3", magic.MP3).
alias("audio/x-mpeg", "audio/mp3")
flac = newMIME("audio/flac", ".flac", magic.Flac)
midi = newMIME("audio/midi", ".midi", magic.Midi).
@@ -192,7 +200,8 @@ var (
avi = newMIME("video/x-msvideo", ".avi", magic.Avi).
alias("video/avi", "video/msvideo")
flv = newMIME("video/x-flv", ".flv", magic.Flv)
mkv = newMIME("video/x-matroska", ".mkv", magic.Mkv)
mkv = newMIME("video/matroska", ".mkv", magic.Mkv).
alias("video/x-matroska")
asf = newMIME("video/x-ms-asf", ".asf", magic.Asf).
alias("video/asf", "video/x-ms-wmv")
rmvb = newMIME("application/vnd.rn-realmedia-vbr", ".rmvb", magic.Rmvb)
@@ -242,8 +251,8 @@ var (
odc = newMIME("application/vnd.oasis.opendocument.chart", ".odc", magic.Odc).
alias("application/x-vnd.oasis.opendocument.chart")
sxc = newMIME("application/vnd.sun.xml.calc", ".sxc", magic.Sxc)
rar = newMIME("application/x-rar-compressed", ".rar", magic.RAR).
alias("application/x-rar")
rar = newMIME("application/vnd.rar", ".rar", magic.RAR).
alias("application/x-rar-compressed", "application/x-rar")
djvu = newMIME("image/vnd.djvu", ".djvu", magic.DjVu)
mobi = newMIME("application/x-mobipocket-ebook", ".mobi", magic.Mobi)
lit = newMIME("application/x-ms-reader", ".lit", magic.Lit)
@@ -294,4 +303,6 @@ var (
hlp = newMIME("application/x-os2-hlp", ".hlp", magic.Hlp)
fm = newMIME("application/vnd.framemaker", ".fm", magic.FrameMaker)
bufr = newMIME("application/bufr", ".bufr", magic.BUFR)
gedcom = newMIME("text/vnd.familysearch.gedcom", ".ged", magic.GEDCOM)
pcap = newMIME("application/vnd.tcpdump.pcap", ".pcap", magic.Pcap)
)

4
vendor/modules.txt vendored
View File

@@ -430,14 +430,16 @@ github.com/felixge/httpsnoop
## explicit; go 1.23
github.com/fsnotify/fsnotify
github.com/fsnotify/fsnotify/internal
# github.com/gabriel-vasile/mimetype v1.4.13
# github.com/gabriel-vasile/mimetype v1.4.15
## explicit; go 1.21
github.com/gabriel-vasile/mimetype
github.com/gabriel-vasile/mimetype/internal/cdf
github.com/gabriel-vasile/mimetype/internal/charset
github.com/gabriel-vasile/mimetype/internal/csv
github.com/gabriel-vasile/mimetype/internal/json
github.com/gabriel-vasile/mimetype/internal/magic
github.com/gabriel-vasile/mimetype/internal/markup
github.com/gabriel-vasile/mimetype/internal/mp3
github.com/gabriel-vasile/mimetype/internal/scan
# github.com/gdexlab/go-render v1.0.1
## explicit