validate inputs

This commit is contained in:
Michael McGuinness
2024-12-23 16:49:53 +00:00
parent a54a70047b
commit 1a5fab87c6
223 changed files with 36539 additions and 643 deletions
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package winterm
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package winterm
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package winterm
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package winterm
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package winterm
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package winterm
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package winterm
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package winterm
+19 -20
View File
@@ -15,26 +15,26 @@ import (
// When defining struct types. the `document` struct tag can be used to control how the value will be
// marshaled into the resulting protocol document.
//
// // Field is ignored
// Field int `document:"-"`
// // Field is ignored
// Field int `document:"-"`
//
// // Field object of key "myName"
// Field int `document:"myName"`
// // Field object of key "myName"
// Field int `document:"myName"`
//
// // Field object key of key "myName", and
// // Field is omitted if the field is a zero value for the type.
// Field int `document:"myName,omitempty"`
// // Field object key of key "myName", and
// // Field is omitted if the field is a zero value for the type.
// Field int `document:"myName,omitempty"`
//
// // Field object key of "Field", and
// // Field is omitted if the field is a zero value for the type.
// Field int `document:",omitempty"`
// // Field object key of "Field", and
// // Field is omitted if the field is a zero value for the type.
// Field int `document:",omitempty"`
//
// All struct fields, including anonymous fields, are marshaled unless the
// any of the following conditions are meet.
//
// - the field is not exported
// - document field tag is "-"
// - document field tag specifies "omitempty", and is a zero value.
// - the field is not exported
// - document field tag is "-"
// - document field tag specifies "omitempty", and is a zero value.
//
// Pointer and interface values are encoded as the value pointed to or
// contained in the interface. A nil value encodes as a null
@@ -63,13 +63,12 @@ type Marshaler interface {
//
// Both generic interface{} and concrete types are valid unmarshal destination types. When unmarshaling a document
// into an empty interface the Unmarshaler will store one of these values:
//
// bool, for boolean values
// document.Number, for arbitrary-precision numbers (int64, float64, big.Int, big.Float)
// string, for string values
// []interface{}, for array values
// map[string]interface{}, for objects
// nil, for null values
// bool, for boolean values
// document.Number, for arbitrary-precision numbers (int64, float64, big.Int, big.Float)
// string, for string values
// []interface{}, for array values
// map[string]interface{}, for objects
// nil, for null values
//
// When unmarshaling, any error that occurs will halt the unmarshal and return the error.
type Unmarshaler interface {
+5 -5
View File
@@ -4,21 +4,21 @@ shape serializer function in which a xml.Value will be passed around.
Resources followed: https://smithy.io/2.0/spec/protocol-traits.html#xml-bindings
# Member Element
Member Element
Member element should be used to encode xml shapes into xml elements except for flattened xml shapes. Member element
write their own element start tag. These elements should always be closed.
# Flattened Element
Flattened Element
Flattened element should be used to encode shapes marked with flattened trait into xml elements. Flattened element
do not write a start tag, and thus should not be closed.
# Simple types encoding
Simple types encoding
All simple type methods on value such as String(), Long() etc; auto close the associated member element.
# Array
Array
Array returns the collection encoder. It has two modes, wrapped and flattened encoding.
@@ -32,7 +32,7 @@ If a shape is marked as flattened, Array() will use the shape element name as wr
<flattenedAarray>apple</flattenedArray><flattenedArray>tree</flattenedArray>
# Map
Map
Map is the map encoder. It has two modes, wrapped and flattened encoding.
+10 -10
View File
@@ -14,7 +14,7 @@
// handler. A handler could be something like an HTTP Client that round trips an
// API operation over HTTP.
//
// # Smithy Middleware Stack
// Smithy Middleware Stack
//
// A Stack is a collection of middleware that wrap a handler. The stack can be
// broken down into discreet steps. Each step may contain zero or more middleware
@@ -48,20 +48,20 @@
// the request message. Deserializes the response into a structured type or
// error above stacks can react to.
//
// # Adding Middleware to a Stack Step
// Adding Middleware to a Stack Step
//
// Middleware can be added to a step front or back, or relative, by name, to an
// existing middleware in that stack. If a middleware does not have a name, a
// unique name will be generated at the middleware and be added to the step.
//
// // Create middleware stack
// stack := middleware.NewStack()
// // Create middleware stack
// stack := middleware.NewStack()
//
// // Add middleware to stack steps
// stack.Initialize.Add(paramValidationMiddleware, middleware.After)
// stack.Serialize.Add(marshalOperationFoo, middleware.After)
// stack.Deserialize.Add(unmarshalOperationFoo, middleware.After)
// // Add middleware to stack steps
// stack.Initialize.Add(paramValidationMiddleware, middleware.After)
// stack.Serialize.Add(marshalOperationFoo, middleware.After)
// stack.Deserialize.Add(unmarshalOperationFoo, middleware.After)
//
// // Invoke middleware on handler.
// resp, err := stack.HandleMiddleware(ctx, req.Input, clientHandler)
// // Invoke middleware on handler.
// resp, err := stack.HandleMiddleware(ctx, req.Input, clientHandler)
package middleware
+2 -2
View File
@@ -13,7 +13,7 @@ import (
// Steps are composed as middleware around the underlying handler in the
// following order:
//
// Initialize -> Serialize -> Build -> Finalize -> Deserialize -> Handler
// Initialize -> Serialize -> Build -> Finalize -> Deserialize -> Handler
//
// Any middleware within the chain may choose to stop and return an error or
// response. Since the middleware decorate the handler like a call stack, each
@@ -21,7 +21,7 @@ import (
// Middleware that does not need to react to an input, or result must forward
// along the input down the chain, or return the result back up the chain.
//
// Initialize <- Serialize -> Build -> Finalize <- Deserialize <- Handler
// Initialize <- Serialize -> Build -> Finalize <- Deserialize <- Handler
type Stack struct {
// Initialize prepares the input, and sets any default parameters as
// needed, (e.g. idempotency token, and presigned URLs).
+18 -20
View File
@@ -11,17 +11,17 @@ period for each retry attempt using a randomization function that grows exponent
NextBackOff() is calculated using the following formula:
randomized interval =
RetryInterval * (random value in range [1 - RandomizationFactor, 1 + RandomizationFactor])
randomized interval =
RetryInterval * (random value in range [1 - RandomizationFactor, 1 + RandomizationFactor])
In other words NextBackOff() will range between the randomization factor
percentage below and above the retry interval.
For example, given the following parameters:
RetryInterval = 2
RandomizationFactor = 0.5
Multiplier = 2
RetryInterval = 2
RandomizationFactor = 0.5
Multiplier = 2
the actual backoff period used in the next retry attempt will range between 1 and 3 seconds,
multiplied by the exponential, that is, between 2 and 6 seconds.
@@ -36,18 +36,18 @@ The elapsed time can be reset by calling Reset().
Example: Given the following default arguments, for 10 tries the sequence will be,
and assuming we go over the MaxElapsedTime on the 10th try:
Request # RetryInterval (seconds) Randomized Interval (seconds)
Request # RetryInterval (seconds) Randomized Interval (seconds)
1 0.5 [0.25, 0.75]
2 0.75 [0.375, 1.125]
3 1.125 [0.562, 1.687]
4 1.687 [0.8435, 2.53]
5 2.53 [1.265, 3.795]
6 3.795 [1.897, 5.692]
7 5.692 [2.846, 8.538]
8 8.538 [4.269, 12.807]
9 12.807 [6.403, 19.210]
10 19.210 backoff.Stop
1 0.5 [0.25, 0.75]
2 0.75 [0.375, 1.125]
3 1.125 [0.562, 1.687]
4 1.687 [0.8435, 2.53]
5 2.53 [1.265, 3.795]
6 3.795 [1.897, 5.692]
7 5.692 [2.846, 8.538]
8 8.538 [4.269, 12.807]
9 12.807 [6.403, 19.210]
10 19.210 backoff.Stop
Note: Implementation is not thread-safe.
*/
@@ -167,8 +167,7 @@ func (b *ExponentialBackOff) Reset() {
}
// NextBackOff calculates the next backoff interval using the formula:
//
// Randomized interval = RetryInterval * (1 ± RandomizationFactor)
// Randomized interval = RetryInterval * (1 ± RandomizationFactor)
func (b *ExponentialBackOff) NextBackOff() time.Duration {
// Make sure we have not gone over the maximum elapsed time.
elapsed := b.GetElapsedTime()
@@ -201,8 +200,7 @@ func (b *ExponentialBackOff) incrementCurrentInterval() {
}
// Returns a random value from the following interval:
//
// [currentInterval - randomizationFactor * currentInterval, currentInterval + randomizationFactor * currentInterval].
// [currentInterval - randomizationFactor * currentInterval, currentInterval + randomizationFactor * currentInterval].
func getRandomValueFromInterval(randomizationFactor, random float64, currentInterval time.Duration) time.Duration {
if randomizationFactor == 0 {
return currentInterval // make sure no randomness is used when randomizationFactor is 0.
-1
View File
@@ -18,7 +18,6 @@
// tag is deprecated and thus should not be used.
// Go versions prior to 1.4 are disabled because they use a different layout
// for interfaces which make the implementation of unsafeReflectValue more complex.
//go:build !js && !appengine && !safe && !disableunsafe && go1.4
// +build !js,!appengine,!safe,!disableunsafe,go1.4
package spew
-1
View File
@@ -16,7 +16,6 @@
// when the code is running on Google App Engine, compiled by GopherJS, or
// "-tags safe" is added to the go build command line. The "disableunsafe"
// tag is deprecated and thus should not be used.
//go:build js || appengine || safe || disableunsafe || !go1.4
// +build js appengine safe disableunsafe !go1.4
package spew
+15 -15
View File
@@ -254,15 +254,15 @@ pointer addresses used to indirect to the final value. It provides the
following features over the built-in printing facilities provided by the fmt
package:
- Pointers are dereferenced and followed
- Circular data structures are detected and handled properly
- Custom Stringer/error interfaces are optionally invoked, including
on unexported types
- Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
- Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output
* Pointers are dereferenced and followed
* Circular data structures are detected and handled properly
* Custom Stringer/error interfaces are optionally invoked, including
on unexported types
* Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
* Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output
The configuration options are controlled by modifying the public members
of c. See ConfigState for options documentation.
@@ -295,12 +295,12 @@ func (c *ConfigState) convertArgs(args []interface{}) (formatters []interface{})
// NewDefaultConfig returns a ConfigState with the following default settings.
//
// Indent: " "
// MaxDepth: 0
// DisableMethods: false
// DisablePointerMethods: false
// ContinueOnMethod: false
// SortKeys: false
// Indent: " "
// MaxDepth: 0
// DisableMethods: false
// DisablePointerMethods: false
// ContinueOnMethod: false
// SortKeys: false
func NewDefaultConfig() *ConfigState {
return &ConfigState{Indent: " "}
}
+61 -67
View File
@@ -21,36 +21,35 @@ debugging.
A quick overview of the additional features spew provides over the built-in
printing facilities for Go data types are as follows:
- Pointers are dereferenced and followed
- Circular data structures are detected and handled properly
- Custom Stringer/error interfaces are optionally invoked, including
on unexported types
- Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
- Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output (only when using
Dump style)
* Pointers are dereferenced and followed
* Circular data structures are detected and handled properly
* Custom Stringer/error interfaces are optionally invoked, including
on unexported types
* Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
* Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output (only when using
Dump style)
There are two different approaches spew allows for dumping Go data structures:
- Dump style which prints with newlines, customizable indentation,
and additional debug information such as types and all pointer addresses
used to indirect to the final value
- A custom Formatter interface that integrates cleanly with the standard fmt
package and replaces %v, %+v, %#v, and %#+v to provide inline printing
similar to the default %v while providing the additional functionality
outlined above and passing unsupported format verbs such as %x and %q
along to fmt
* Dump style which prints with newlines, customizable indentation,
and additional debug information such as types and all pointer addresses
used to indirect to the final value
* A custom Formatter interface that integrates cleanly with the standard fmt
package and replaces %v, %+v, %#v, and %#+v to provide inline printing
similar to the default %v while providing the additional functionality
outlined above and passing unsupported format verbs such as %x and %q
along to fmt
# Quick Start
Quick Start
This section demonstrates how to quickly get started with spew. See the
sections below for further details on formatting and configuration options.
To dump a variable with full newlines, indentation, type, and pointer
information use Dump, Fdump, or Sdump:
spew.Dump(myVar1, myVar2, ...)
spew.Fdump(someWriter, myVar1, myVar2, ...)
str := spew.Sdump(myVar1, myVar2, ...)
@@ -59,13 +58,12 @@ Alternatively, if you would prefer to use format strings with a compacted inline
printing style, use the convenience wrappers Printf, Fprintf, etc with
%v (most compact), %+v (adds pointer addresses), %#v (adds types), or
%#+v (adds types and pointer addresses):
spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2)
spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
spew.Fprintf(someWriter, "myVar1: %v -- myVar2: %+v", myVar1, myVar2)
spew.Fprintf(someWriter, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4)
# Configuration Options
Configuration Options
Configuration of spew is handled by fields in the ConfigState type. For
convenience, all of the top-level functions use a global state available
@@ -76,52 +74,51 @@ equivalent to the top-level functions. This allows concurrent configuration
options. See the ConfigState documentation for more details.
The following configuration options are available:
* Indent
String to use for each indentation level for Dump functions.
It is a single space by default. A popular alternative is "\t".
- Indent
String to use for each indentation level for Dump functions.
It is a single space by default. A popular alternative is "\t".
* MaxDepth
Maximum number of levels to descend into nested data structures.
There is no limit by default.
- MaxDepth
Maximum number of levels to descend into nested data structures.
There is no limit by default.
* DisableMethods
Disables invocation of error and Stringer interface methods.
Method invocation is enabled by default.
- DisableMethods
Disables invocation of error and Stringer interface methods.
Method invocation is enabled by default.
* DisablePointerMethods
Disables invocation of error and Stringer interface methods on types
which only accept pointer receivers from non-pointer variables.
Pointer method invocation is enabled by default.
- DisablePointerMethods
Disables invocation of error and Stringer interface methods on types
which only accept pointer receivers from non-pointer variables.
Pointer method invocation is enabled by default.
* DisablePointerAddresses
DisablePointerAddresses specifies whether to disable the printing of
pointer addresses. This is useful when diffing data structures in tests.
- DisablePointerAddresses
DisablePointerAddresses specifies whether to disable the printing of
pointer addresses. This is useful when diffing data structures in tests.
* DisableCapacities
DisableCapacities specifies whether to disable the printing of
capacities for arrays, slices, maps and channels. This is useful when
diffing data structures in tests.
- DisableCapacities
DisableCapacities specifies whether to disable the printing of
capacities for arrays, slices, maps and channels. This is useful when
diffing data structures in tests.
* ContinueOnMethod
Enables recursion into types after invoking error and Stringer interface
methods. Recursion after method invocation is disabled by default.
- ContinueOnMethod
Enables recursion into types after invoking error and Stringer interface
methods. Recursion after method invocation is disabled by default.
* SortKeys
Specifies map keys should be sorted before being printed. Use
this to have a more deterministic, diffable output. Note that
only native types (bool, int, uint, floats, uintptr and string)
and types which implement error or Stringer interfaces are
supported with other types sorted according to the
reflect.Value.String() output which guarantees display
stability. Natural map order is used by default.
- SortKeys
Specifies map keys should be sorted before being printed. Use
this to have a more deterministic, diffable output. Note that
only native types (bool, int, uint, floats, uintptr and string)
and types which implement error or Stringer interfaces are
supported with other types sorted according to the
reflect.Value.String() output which guarantees display
stability. Natural map order is used by default.
* SpewKeys
Specifies that, as a last resort attempt, map keys should be
spewed to strings and sorted by those strings. This is only
considered if SortKeys is true.
- SpewKeys
Specifies that, as a last resort attempt, map keys should be
spewed to strings and sorted by those strings. This is only
considered if SortKeys is true.
# Dump Usage
Dump Usage
Simply call spew.Dump with a list of variables you want to dump:
@@ -136,7 +133,7 @@ A third option is to call spew.Sdump to get the formatted output as a string:
str := spew.Sdump(myVar1, myVar2, ...)
# Sample Dump Output
Sample Dump Output
See the Dump example for details on the setup of the types and variables being
shown here.
@@ -153,14 +150,13 @@ shown here.
Byte (and uint8) arrays and slices are displayed uniquely like the hexdump -C
command as shown.
([]uint8) (len=32 cap=32) {
00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 |............... |
00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 |!"#$%&'()*+,-./0|
00000020 31 32 |12|
}
# Custom Formatter
Custom Formatter
Spew provides a custom formatter that implements the fmt.Formatter interface
so that it integrates cleanly with standard fmt package printing functions. The
@@ -174,7 +170,7 @@ standard fmt package for formatting. In addition, the custom formatter ignores
the width and precision arguments (however they will still work on the format
specifiers not handled by the custom formatter).
# Custom Formatter Usage
Custom Formatter Usage
The simplest way to make use of the spew custom formatter is to call one of the
convenience functions such as spew.Printf, spew.Println, or spew.Printf. The
@@ -188,17 +184,15 @@ functions have syntax you are most likely already familiar with:
See the Index for the full list convenience functions.
# Sample Formatter Output
Sample Formatter Output
Double pointer to a uint8:
%v: <**>5
%+v: <**>(0xf8400420d0->0xf8400420c8)5
%#v: (**uint8)5
%#+v: (**uint8)(0xf8400420d0->0xf8400420c8)5
Pointer to circular struct with a uint8 field and a pointer to itself:
%v: <*>{1 <*><shown>}
%+v: <*>(0xf84003e260){ui8:1 c:<*>(0xf84003e260)<shown>}
%#v: (*main.circular){ui8:(uint8)1 c:(*main.circular)<shown>}
@@ -207,7 +201,7 @@ Pointer to circular struct with a uint8 field and a pointer to itself:
See the Printf example for details on the setup of variables being shown
here.
# Errors
Errors
Since it is possible for custom Stringer/error interfaces to panic, spew
detects them and handles them internally by printing the panic information
+9 -9
View File
@@ -488,15 +488,15 @@ pointer addresses used to indirect to the final value. It provides the
following features over the built-in printing facilities provided by the fmt
package:
- Pointers are dereferenced and followed
- Circular data structures are detected and handled properly
- Custom Stringer/error interfaces are optionally invoked, including
on unexported types
- Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
- Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output
* Pointers are dereferenced and followed
* Circular data structures are detected and handled properly
* Custom Stringer/error interfaces are optionally invoked, including
on unexported types
* Custom types which only implement the Stringer/error interfaces via
a pointer receiver are optionally invoked when passing non-pointer
variables
* Byte arrays and slices are dumped like the hexdump -C command which
includes offsets, byte values in hex, and ASCII output
The configuration options are controlled by an exported package global,
spew.Config. See ConfigState for options documentation.
-2
View File
@@ -1,6 +1,4 @@
//go:build go1.8
// +build go1.8
// Code generated by "httpsnoop/codegen"; DO NOT EDIT.
package httpsnoop
-2
View File
@@ -1,6 +1,4 @@
//go:build !go1.8
// +build !go1.8
// Code generated by "httpsnoop/codegen"; DO NOT EDIT.
package httpsnoop
@@ -0,0 +1 @@
testdata/* linguist-vendored
+76
View File
@@ -0,0 +1,76 @@
# Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, sex characteristics, gender identity and expression,
level of experience, education, socio-economic status, nationality, personal
appearance, race, religion, or sexual identity and orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at vasile.gabriel@email.com. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
[homepage]: https://www.contributor-covenant.org
For answers to common questions about this code of conduct, see
https://www.contributor-covenant.org/faq
+12
View File
@@ -0,0 +1,12 @@
## Contribute
Contributions to **mimetype** are welcome. If you find an issue and you consider
contributing, you can use the [Github issues tracker](https://github.com/gabriel-vasile/mimetype/issues)
in order to report it, or better yet, open a pull request.
Code contributions must respect these rules:
- code must be test covered
- code must be formatted using gofmt tool
- exported names must be documented
**Important**: By submitting a pull request, you agree to allow the project
owner to license your work under the same license as that used by the project.
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2018-2020 Gabriel Vasile
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+105
View File
@@ -0,0 +1,105 @@
<h1 align="center">
mimetype
</h1>
<h4 align="center">
A package for detecting MIME types and extensions based on magic numbers
</h4>
<h6 align="center">
Goroutine safe, extensible, no C bindings
</h6>
<p align="center">
<a href="https://pkg.go.dev/github.com/gabriel-vasile/mimetype">
<img alt="Go Reference" src="https://pkg.go.dev/badge/github.com/gabriel-vasile/mimetype.svg">
</a>
<a href="https://goreportcard.com/report/github.com/gabriel-vasile/mimetype">
<img alt="Go report card" src="https://goreportcard.com/badge/github.com/gabriel-vasile/mimetype">
</a>
<a href="https://codecov.io/gh/gabriel-vasile/mimetype">
<img alt="Code coverage" src="https://codecov.io/gh/gabriel-vasile/mimetype/branch/master/graph/badge.svg?token=qcfJF1kkl2"/>
</a>
<a href="LICENSE">
<img alt="License" src="https://img.shields.io/badge/License-MIT-green.svg">
</a>
</p>
## Features
- fast and precise MIME type and file extension detection
- long list of [supported MIME types](supported_mimes.md)
- possibility to [extend](https://pkg.go.dev/github.com/gabriel-vasile/mimetype#example-package-Extend) with other file formats
- common file formats are prioritized
- [text vs. binary files differentiation](https://pkg.go.dev/github.com/gabriel-vasile/mimetype#example-package-TextVsBinary)
- safe for concurrent usage
## Install
```bash
go get github.com/gabriel-vasile/mimetype
```
## Usage
```go
mtype := mimetype.Detect([]byte)
// OR
mtype, err := mimetype.DetectReader(io.Reader)
// OR
mtype, err := mimetype.DetectFile("/path/to/file")
fmt.Println(mtype.String(), mtype.Extension())
```
See the [runnable Go Playground examples](https://pkg.go.dev/github.com/gabriel-vasile/mimetype#pkg-overview).
## Usage'
Only use libraries like **mimetype** as a last resort. Content type detection
using magic numbers is slow, inaccurate, and non-standard. Most of the times
protocols have methods for specifying such metadata; e.g., `Content-Type` header
in HTTP and SMTP.
## FAQ
Q: My file is in the list of [supported MIME types](supported_mimes.md) but
it is not correctly detected. What should I do?
A: Some file formats (often Microsoft Office documents) keep their signatures
towards the end of the file. Try increasing the number of bytes used for detection
with:
```go
mimetype.SetLimit(1024*1024) // Set limit to 1MB.
// or
mimetype.SetLimit(0) // No limit, whole file content used.
mimetype.DetectFile("file.doc")
```
If increasing the limit does not help, please
[open an issue](https://github.com/gabriel-vasile/mimetype/issues/new?assignees=&labels=&template=mismatched-mime-type-detected.md&title=).
## Structure
**mimetype** uses a hierarchical structure to keep the MIME type detection logic.
This reduces the number of calls needed for detecting the file type. The reason
behind this choice is that there are file formats used as containers for other
file formats. For example, Microsoft Office files are just zip archives,
containing specific metadata files. Once a file has been identified as a
zip, there is no need to check if it is a text file, but it is worth checking if
it is an Microsoft Office file.
To prevent loading entire files into memory, when detecting from a
[reader](https://pkg.go.dev/github.com/gabriel-vasile/mimetype#DetectReader)
or from a [file](https://pkg.go.dev/github.com/gabriel-vasile/mimetype#DetectFile)
**mimetype** limits itself to reading only the header of the input.
<div align="center">
<img alt="structure" src="https://github.com/gabriel-vasile/mimetype/blob/420a05228c6a6efbb6e6f080168a25663414ff36/mimetype.gif?raw=true" width="88%">
</div>
## Performance
Thanks to the hierarchical structure, searching for common formats first,
and limiting itself to file headers, **mimetype** matches the performance of
stdlib `http.DetectContentType` while outperforming the alternative package.
```bash
mimetype http.DetectContentType filetype
BenchmarkMatchTar-24 250 ns/op 400 ns/op 3778 ns/op
BenchmarkMatchZip-24 524 ns/op 351 ns/op 4884 ns/op
BenchmarkMatchJpeg-24 103 ns/op 228 ns/op 839 ns/op
BenchmarkMatchGif-24 139 ns/op 202 ns/op 751 ns/op
BenchmarkMatchPng-24 165 ns/op 221 ns/op 1176 ns/op
```
## Contributing
See [CONTRIBUTING.md](CONTRIBUTING.md).
+309
View File
@@ -0,0 +1,309 @@
package charset
import (
"bytes"
"encoding/xml"
"strings"
"unicode/utf8"
"golang.org/x/net/html"
)
const (
F = 0 /* character never appears in text */
T = 1 /* character appears in plain ASCII text */
I = 2 /* character appears in ISO-8859 text */
X = 3 /* character appears in non-ISO extended ASCII (Mac, IBM PC) */
)
var (
boms = []struct {
bom []byte
enc string
}{
{[]byte{0xEF, 0xBB, 0xBF}, "utf-8"},
{[]byte{0x00, 0x00, 0xFE, 0xFF}, "utf-32be"},
{[]byte{0xFF, 0xFE, 0x00, 0x00}, "utf-32le"},
{[]byte{0xFE, 0xFF}, "utf-16be"},
{[]byte{0xFF, 0xFE}, "utf-16le"},
}
// https://github.com/file/file/blob/fa93fb9f7d21935f1c7644c47d2975d31f12b812/src/encoding.c#L241
textChars = [256]byte{
/* BEL BS HT LF VT FF CR */
F, F, F, F, F, F, F, T, T, T, T, T, T, T, F, F, /* 0x0X */
/* ESC */
F, F, F, F, F, F, F, F, F, F, F, T, F, F, F, F, /* 0x1X */
T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x2X */
T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x3X */
T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x4X */
T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x5X */
T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x6X */
T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, F, /* 0x7X */
/* NEL */
X, X, X, X, X, T, X, X, X, X, X, X, X, X, X, X, /* 0x8X */
X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, /* 0x9X */
I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xaX */
I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xbX */
I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xcX */
I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xdX */
I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xeX */
I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xfX */
}
)
// FromBOM returns the charset declared in the BOM of content.
func FromBOM(content []byte) string {
for _, b := range boms {
if bytes.HasPrefix(content, b.bom) {
return b.enc
}
}
return ""
}
// FromPlain returns the charset of a plain text. It relies on BOM presence
// and it falls back on checking each byte in content.
func FromPlain(content []byte) string {
if len(content) == 0 {
return ""
}
if cset := FromBOM(content); cset != "" {
return cset
}
origContent := content
// Try to detect UTF-8.
// First eliminate any partial rune at the end.
for i := len(content) - 1; i >= 0 && i > len(content)-4; i-- {
b := content[i]
if b < 0x80 {
break
}
if utf8.RuneStart(b) {
content = content[:i]
break
}
}
hasHighBit := false
for _, c := range content {
if c >= 0x80 {
hasHighBit = true
break
}
}
if hasHighBit && utf8.Valid(content) {
return "utf-8"
}
// ASCII is a subset of UTF8. Follow W3C recommendation and replace with UTF8.
if ascii(origContent) {
return "utf-8"
}
return latin(origContent)
}
func latin(content []byte) string {
hasControlBytes := false
for _, b := range content {
t := textChars[b]
if t != T && t != I {
return ""
}
if b >= 0x80 && b <= 0x9F {
hasControlBytes = true
}
}
// Code range 0x80 to 0x9F is reserved for control characters in ISO-8859-1
// (so-called C1 Controls). Windows 1252, however, has printable punctuation
// characters in this range.
if hasControlBytes {
return "windows-1252"
}
return "iso-8859-1"
}
func ascii(content []byte) bool {
for _, b := range content {
if textChars[b] != T {
return false
}
}
return true
}
// FromXML returns the charset of an XML document. It relies on the XML
// header <?xml version="1.0" encoding="UTF-8"?> and falls back on the plain
// text content.
func FromXML(content []byte) string {
if cset := fromXML(content); cset != "" {
return cset
}
return FromPlain(content)
}
func fromXML(content []byte) string {
content = trimLWS(content)
dec := xml.NewDecoder(bytes.NewReader(content))
rawT, err := dec.RawToken()
if err != nil {
return ""
}
t, ok := rawT.(xml.ProcInst)
if !ok {
return ""
}
return strings.ToLower(xmlEncoding(string(t.Inst)))
}
// FromHTML returns the charset of an HTML document. It first looks if a BOM is
// present and if so uses it to determine the charset. If no BOM is present,
// it relies on the meta tag <meta charset="UTF-8"> and falls back on the
// plain text content.
func FromHTML(content []byte) string {
if cset := FromBOM(content); cset != "" {
return cset
}
if cset := fromHTML(content); cset != "" {
return cset
}
return FromPlain(content)
}
func fromHTML(content []byte) string {
z := html.NewTokenizer(bytes.NewReader(content))
for {
switch z.Next() {
case html.ErrorToken:
return ""
case html.StartTagToken, html.SelfClosingTagToken:
tagName, hasAttr := z.TagName()
if !bytes.Equal(tagName, []byte("meta")) {
continue
}
attrList := make(map[string]bool)
gotPragma := false
const (
dontKnow = iota
doNeedPragma
doNotNeedPragma
)
needPragma := dontKnow
name := ""
for hasAttr {
var key, val []byte
key, val, hasAttr = z.TagAttr()
ks := string(key)
if attrList[ks] {
continue
}
attrList[ks] = true
for i, c := range val {
if 'A' <= c && c <= 'Z' {
val[i] = c + 0x20
}
}
switch ks {
case "http-equiv":
if bytes.Equal(val, []byte("content-type")) {
gotPragma = true
}
case "content":
name = fromMetaElement(string(val))
if name != "" {
needPragma = doNeedPragma
}
case "charset":
name = string(val)
needPragma = doNotNeedPragma
}
}
if needPragma == dontKnow || needPragma == doNeedPragma && !gotPragma {
continue
}
if strings.HasPrefix(name, "utf-16") {
name = "utf-8"
}
return name
}
}
}
func fromMetaElement(s string) string {
for s != "" {
csLoc := strings.Index(s, "charset")
if csLoc == -1 {
return ""
}
s = s[csLoc+len("charset"):]
s = strings.TrimLeft(s, " \t\n\f\r")
if !strings.HasPrefix(s, "=") {
continue
}
s = s[1:]
s = strings.TrimLeft(s, " \t\n\f\r")
if s == "" {
return ""
}
if q := s[0]; q == '"' || q == '\'' {
s = s[1:]
closeQuote := strings.IndexRune(s, rune(q))
if closeQuote == -1 {
return ""
}
return s[:closeQuote]
}
end := strings.IndexAny(s, "; \t\n\f\r")
if end == -1 {
end = len(s)
}
return s[:end]
}
return ""
}
func xmlEncoding(s string) string {
param := "encoding="
idx := strings.Index(s, param)
if idx == -1 {
return ""
}
v := s[idx+len(param):]
if v == "" {
return ""
}
if v[0] != '\'' && v[0] != '"' {
return ""
}
idx = strings.IndexRune(v[1:], rune(v[0]))
if idx == -1 {
return ""
}
return v[1 : idx+1]
}
// trimLWS trims whitespace from beginning of the input.
// TODO: find a way to call trimLWS once per detection instead of once in each
// detector which needs the trimmed input.
func trimLWS(in []byte) []byte {
firstNonWS := 0
for ; firstNonWS < len(in) && isWS(in[firstNonWS]); firstNonWS++ {
}
return in[firstNonWS:]
}
func isWS(b byte) bool {
return b == '\t' || b == '\n' || b == '\x0c' || b == '\r' || b == ' '
}
+544
View File
@@ -0,0 +1,544 @@
// Copyright (c) 2009 The Go Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Package json provides a JSON value parser state machine.
// This package is almost entirely copied from the Go stdlib.
// Changes made to it permit users of the package to tell
// if some slice of bytes is a valid beginning of a json string.
package json
import (
"fmt"
)
type (
scanStatus int
)
const (
parseObjectKey = iota // parsing object key (before colon)
parseObjectValue // parsing object value (after colon)
parseArrayValue // parsing array value
scanContinue scanStatus = iota // uninteresting byte
scanBeginLiteral // end implied by next result != scanContinue
scanBeginObject // begin object
scanObjectKey // just finished object key (string)
scanObjectValue // just finished non-last object value
scanEndObject // end object (implies scanObjectValue if possible)
scanBeginArray // begin array
scanArrayValue // just finished array value
scanEndArray // end array (implies scanArrayValue if possible)
scanSkipSpace // space byte; can skip; known to be last "continue" result
scanEnd // top-level value ended *before* this byte; known to be first "stop" result
scanError // hit an error, scanner.err.
// This limits the max nesting depth to prevent stack overflow.
// This is permitted by https://tools.ietf.org/html/rfc7159#section-9
maxNestingDepth = 10000
)
type (
scanner struct {
step func(*scanner, byte) scanStatus
parseState []int
endTop bool
err error
index int
}
)
// Scan returns the number of bytes scanned and if there was any error
// in trying to reach the end of data.
func Scan(data []byte) (int, error) {
s := &scanner{}
_ = checkValid(data, s)
return s.index, s.err
}
// checkValid verifies that data is valid JSON-encoded data.
// scan is passed in for use by checkValid to avoid an allocation.
func checkValid(data []byte, scan *scanner) error {
scan.reset()
for _, c := range data {
scan.index++
if scan.step(scan, c) == scanError {
return scan.err
}
}
if scan.eof() == scanError {
return scan.err
}
return nil
}
func isSpace(c byte) bool {
return c == ' ' || c == '\t' || c == '\r' || c == '\n'
}
func (s *scanner) reset() {
s.step = stateBeginValue
s.parseState = s.parseState[0:0]
s.err = nil
}
// eof tells the scanner that the end of input has been reached.
// It returns a scan status just as s.step does.
func (s *scanner) eof() scanStatus {
if s.err != nil {
return scanError
}
if s.endTop {
return scanEnd
}
s.step(s, ' ')
if s.endTop {
return scanEnd
}
if s.err == nil {
s.err = fmt.Errorf("unexpected end of JSON input")
}
return scanError
}
// pushParseState pushes a new parse state p onto the parse stack.
// an error state is returned if maxNestingDepth was exceeded, otherwise successState is returned.
func (s *scanner) pushParseState(c byte, newParseState int, successState scanStatus) scanStatus {
s.parseState = append(s.parseState, newParseState)
if len(s.parseState) <= maxNestingDepth {
return successState
}
return s.error(c, "exceeded max depth")
}
// popParseState pops a parse state (already obtained) off the stack
// and updates s.step accordingly.
func (s *scanner) popParseState() {
n := len(s.parseState) - 1
s.parseState = s.parseState[0:n]
if n == 0 {
s.step = stateEndTop
s.endTop = true
} else {
s.step = stateEndValue
}
}
// stateBeginValueOrEmpty is the state after reading `[`.
func stateBeginValueOrEmpty(s *scanner, c byte) scanStatus {
if c <= ' ' && isSpace(c) {
return scanSkipSpace
}
if c == ']' {
return stateEndValue(s, c)
}
return stateBeginValue(s, c)
}
// stateBeginValue is the state at the beginning of the input.
func stateBeginValue(s *scanner, c byte) scanStatus {
if c <= ' ' && isSpace(c) {
return scanSkipSpace
}
switch c {
case '{':
s.step = stateBeginStringOrEmpty
return s.pushParseState(c, parseObjectKey, scanBeginObject)
case '[':
s.step = stateBeginValueOrEmpty
return s.pushParseState(c, parseArrayValue, scanBeginArray)
case '"':
s.step = stateInString
return scanBeginLiteral
case '-':
s.step = stateNeg
return scanBeginLiteral
case '0': // beginning of 0.123
s.step = state0
return scanBeginLiteral
case 't': // beginning of true
s.step = stateT
return scanBeginLiteral
case 'f': // beginning of false
s.step = stateF
return scanBeginLiteral
case 'n': // beginning of null
s.step = stateN
return scanBeginLiteral
}
if '1' <= c && c <= '9' { // beginning of 1234.5
s.step = state1
return scanBeginLiteral
}
return s.error(c, "looking for beginning of value")
}
// stateBeginStringOrEmpty is the state after reading `{`.
func stateBeginStringOrEmpty(s *scanner, c byte) scanStatus {
if c <= ' ' && isSpace(c) {
return scanSkipSpace
}
if c == '}' {
n := len(s.parseState)
s.parseState[n-1] = parseObjectValue
return stateEndValue(s, c)
}
return stateBeginString(s, c)
}
// stateBeginString is the state after reading `{"key": value,`.
func stateBeginString(s *scanner, c byte) scanStatus {
if c <= ' ' && isSpace(c) {
return scanSkipSpace
}
if c == '"' {
s.step = stateInString
return scanBeginLiteral
}
return s.error(c, "looking for beginning of object key string")
}
// stateEndValue is the state after completing a value,
// such as after reading `{}` or `true` or `["x"`.
func stateEndValue(s *scanner, c byte) scanStatus {
n := len(s.parseState)
if n == 0 {
// Completed top-level before the current byte.
s.step = stateEndTop
s.endTop = true
return stateEndTop(s, c)
}
if c <= ' ' && isSpace(c) {
s.step = stateEndValue
return scanSkipSpace
}
ps := s.parseState[n-1]
switch ps {
case parseObjectKey:
if c == ':' {
s.parseState[n-1] = parseObjectValue
s.step = stateBeginValue
return scanObjectKey
}
return s.error(c, "after object key")
case parseObjectValue:
if c == ',' {
s.parseState[n-1] = parseObjectKey
s.step = stateBeginString
return scanObjectValue
}
if c == '}' {
s.popParseState()
return scanEndObject
}
return s.error(c, "after object key:value pair")
case parseArrayValue:
if c == ',' {
s.step = stateBeginValue
return scanArrayValue
}
if c == ']' {
s.popParseState()
return scanEndArray
}
return s.error(c, "after array element")
}
return s.error(c, "")
}
// stateEndTop is the state after finishing the top-level value,
// such as after reading `{}` or `[1,2,3]`.
// Only space characters should be seen now.
func stateEndTop(s *scanner, c byte) scanStatus {
if c != ' ' && c != '\t' && c != '\r' && c != '\n' {
// Complain about non-space byte on next call.
s.error(c, "after top-level value")
}
return scanEnd
}
// stateInString is the state after reading `"`.
func stateInString(s *scanner, c byte) scanStatus {
if c == '"' {
s.step = stateEndValue
return scanContinue
}
if c == '\\' {
s.step = stateInStringEsc
return scanContinue
}
if c < 0x20 {
return s.error(c, "in string literal")
}
return scanContinue
}
// stateInStringEsc is the state after reading `"\` during a quoted string.
func stateInStringEsc(s *scanner, c byte) scanStatus {
switch c {
case 'b', 'f', 'n', 'r', 't', '\\', '/', '"':
s.step = stateInString
return scanContinue
case 'u':
s.step = stateInStringEscU
return scanContinue
}
return s.error(c, "in string escape code")
}
// stateInStringEscU is the state after reading `"\u` during a quoted string.
func stateInStringEscU(s *scanner, c byte) scanStatus {
if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
s.step = stateInStringEscU1
return scanContinue
}
// numbers
return s.error(c, "in \\u hexadecimal character escape")
}
// stateInStringEscU1 is the state after reading `"\u1` during a quoted string.
func stateInStringEscU1(s *scanner, c byte) scanStatus {
if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
s.step = stateInStringEscU12
return scanContinue
}
// numbers
return s.error(c, "in \\u hexadecimal character escape")
}
// stateInStringEscU12 is the state after reading `"\u12` during a quoted string.
func stateInStringEscU12(s *scanner, c byte) scanStatus {
if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
s.step = stateInStringEscU123
return scanContinue
}
// numbers
return s.error(c, "in \\u hexadecimal character escape")
}
// stateInStringEscU123 is the state after reading `"\u123` during a quoted string.
func stateInStringEscU123(s *scanner, c byte) scanStatus {
if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' {
s.step = stateInString
return scanContinue
}
// numbers
return s.error(c, "in \\u hexadecimal character escape")
}
// stateNeg is the state after reading `-` during a number.
func stateNeg(s *scanner, c byte) scanStatus {
if c == '0' {
s.step = state0
return scanContinue
}
if '1' <= c && c <= '9' {
s.step = state1
return scanContinue
}
return s.error(c, "in numeric literal")
}
// state1 is the state after reading a non-zero integer during a number,
// such as after reading `1` or `100` but not `0`.
func state1(s *scanner, c byte) scanStatus {
if '0' <= c && c <= '9' {
s.step = state1
return scanContinue
}
return state0(s, c)
}
// state0 is the state after reading `0` during a number.
func state0(s *scanner, c byte) scanStatus {
if c == '.' {
s.step = stateDot
return scanContinue
}
if c == 'e' || c == 'E' {
s.step = stateE
return scanContinue
}
return stateEndValue(s, c)
}
// stateDot is the state after reading the integer and decimal point in a number,
// such as after reading `1.`.
func stateDot(s *scanner, c byte) scanStatus {
if '0' <= c && c <= '9' {
s.step = stateDot0
return scanContinue
}
return s.error(c, "after decimal point in numeric literal")
}
// stateDot0 is the state after reading the integer, decimal point, and subsequent
// digits of a number, such as after reading `3.14`.
func stateDot0(s *scanner, c byte) scanStatus {
if '0' <= c && c <= '9' {
return scanContinue
}
if c == 'e' || c == 'E' {
s.step = stateE
return scanContinue
}
return stateEndValue(s, c)
}
// stateE is the state after reading the mantissa and e in a number,
// such as after reading `314e` or `0.314e`.
func stateE(s *scanner, c byte) scanStatus {
if c == '+' || c == '-' {
s.step = stateESign
return scanContinue
}
return stateESign(s, c)
}
// stateESign is the state after reading the mantissa, e, and sign in a number,
// such as after reading `314e-` or `0.314e+`.
func stateESign(s *scanner, c byte) scanStatus {
if '0' <= c && c <= '9' {
s.step = stateE0
return scanContinue
}
return s.error(c, "in exponent of numeric literal")
}
// stateE0 is the state after reading the mantissa, e, optional sign,
// and at least one digit of the exponent in a number,
// such as after reading `314e-2` or `0.314e+1` or `3.14e0`.
func stateE0(s *scanner, c byte) scanStatus {
if '0' <= c && c <= '9' {
return scanContinue
}
return stateEndValue(s, c)
}
// stateT is the state after reading `t`.
func stateT(s *scanner, c byte) scanStatus {
if c == 'r' {
s.step = stateTr
return scanContinue
}
return s.error(c, "in literal true (expecting 'r')")
}
// stateTr is the state after reading `tr`.
func stateTr(s *scanner, c byte) scanStatus {
if c == 'u' {
s.step = stateTru
return scanContinue
}
return s.error(c, "in literal true (expecting 'u')")
}
// stateTru is the state after reading `tru`.
func stateTru(s *scanner, c byte) scanStatus {
if c == 'e' {
s.step = stateEndValue
return scanContinue
}
return s.error(c, "in literal true (expecting 'e')")
}
// stateF is the state after reading `f`.
func stateF(s *scanner, c byte) scanStatus {
if c == 'a' {
s.step = stateFa
return scanContinue
}
return s.error(c, "in literal false (expecting 'a')")
}
// stateFa is the state after reading `fa`.
func stateFa(s *scanner, c byte) scanStatus {
if c == 'l' {
s.step = stateFal
return scanContinue
}
return s.error(c, "in literal false (expecting 'l')")
}
// stateFal is the state after reading `fal`.
func stateFal(s *scanner, c byte) scanStatus {
if c == 's' {
s.step = stateFals
return scanContinue
}
return s.error(c, "in literal false (expecting 's')")
}
// stateFals is the state after reading `fals`.
func stateFals(s *scanner, c byte) scanStatus {
if c == 'e' {
s.step = stateEndValue
return scanContinue
}
return s.error(c, "in literal false (expecting 'e')")
}
// stateN is the state after reading `n`.
func stateN(s *scanner, c byte) scanStatus {
if c == 'u' {
s.step = stateNu
return scanContinue
}
return s.error(c, "in literal null (expecting 'u')")
}
// stateNu is the state after reading `nu`.
func stateNu(s *scanner, c byte) scanStatus {
if c == 'l' {
s.step = stateNul
return scanContinue
}
return s.error(c, "in literal null (expecting 'l')")
}
// stateNul is the state after reading `nul`.
func stateNul(s *scanner, c byte) scanStatus {
if c == 'l' {
s.step = stateEndValue
return scanContinue
}
return s.error(c, "in literal null (expecting 'l')")
}
// stateError is the state after reaching a syntax error,
// such as after reading `[1}` or `5.1.2`.
func stateError(s *scanner, c byte) scanStatus {
return scanError
}
// error records an error and switches to the error state.
func (s *scanner) error(c byte, context string) scanStatus {
s.step = stateError
s.err = fmt.Errorf("invalid character <<%c>> %s", c, context)
return scanError
}
+124
View File
@@ -0,0 +1,124 @@
package magic
import (
"bytes"
"encoding/binary"
)
var (
// SevenZ matches a 7z archive.
SevenZ = prefix([]byte{0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C})
// Gzip matches gzip files based on http://www.zlib.org/rfc-gzip.html#header-trailer.
Gzip = prefix([]byte{0x1f, 0x8b})
// Fits matches an Flexible Image Transport System file.
Fits = prefix([]byte{
0x53, 0x49, 0x4D, 0x50, 0x4C, 0x45, 0x20, 0x20, 0x3D, 0x20,
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x54,
})
// Xar matches an eXtensible ARchive format file.
Xar = prefix([]byte{0x78, 0x61, 0x72, 0x21})
// Bz2 matches a bzip2 file.
Bz2 = prefix([]byte{0x42, 0x5A, 0x68})
// Ar matches an ar (Unix) archive file.
Ar = prefix([]byte{0x21, 0x3C, 0x61, 0x72, 0x63, 0x68, 0x3E})
// Deb matches a Debian package file.
Deb = offset([]byte{
0x64, 0x65, 0x62, 0x69, 0x61, 0x6E, 0x2D,
0x62, 0x69, 0x6E, 0x61, 0x72, 0x79,
}, 8)
// Warc matches a Web ARChive file.
Warc = prefix([]byte("WARC/1.0"), []byte("WARC/1.1"))
// Cab matches a Microsoft Cabinet archive file.
Cab = prefix([]byte("MSCF\x00\x00\x00\x00"))
// Xz matches an xz compressed stream based on https://tukaani.org/xz/xz-file-format.txt.
Xz = prefix([]byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00})
// Lzip matches an Lzip compressed file.
Lzip = prefix([]byte{0x4c, 0x5a, 0x49, 0x50})
// RPM matches an RPM or Delta RPM package file.
RPM = prefix([]byte{0xed, 0xab, 0xee, 0xdb}, []byte("drpm"))
// Cpio matches a cpio archive file.
Cpio = prefix([]byte("070707"), []byte("070701"), []byte("070702"))
// RAR matches a RAR archive file.
RAR = prefix([]byte("Rar!\x1A\x07\x00"), []byte("Rar!\x1A\x07\x01\x00"))
)
// InstallShieldCab matches an InstallShield Cabinet archive file.
func InstallShieldCab(raw []byte, _ uint32) bool {
return len(raw) > 7 &&
bytes.Equal(raw[0:4], []byte("ISc(")) &&
raw[6] == 0 &&
(raw[7] == 1 || raw[7] == 2 || raw[7] == 4)
}
// Zstd matches a Zstandard archive file.
func Zstd(raw []byte, limit uint32) bool {
return len(raw) >= 4 &&
(0x22 <= raw[0] && raw[0] <= 0x28 || raw[0] == 0x1E) && // Different Zstandard versions.
bytes.HasPrefix(raw[1:], []byte{0xB5, 0x2F, 0xFD})
}
// CRX matches a Chrome extension file: a zip archive prepended by a package header.
func CRX(raw []byte, limit uint32) bool {
const minHeaderLen = 16
if len(raw) < minHeaderLen || !bytes.HasPrefix(raw, []byte("Cr24")) {
return false
}
pubkeyLen := binary.LittleEndian.Uint32(raw[8:12])
sigLen := binary.LittleEndian.Uint32(raw[12:16])
zipOffset := minHeaderLen + pubkeyLen + sigLen
if uint32(len(raw)) < zipOffset {
return false
}
return Zip(raw[zipOffset:], limit)
}
// Tar matches a (t)ape (ar)chive file.
func Tar(raw []byte, _ uint32) bool {
// The "magic" header field for files in in UStar (POSIX IEEE P1003.1) archives
// has the prefix "ustar". The values of the remaining bytes in this field vary
// by archiver implementation.
if len(raw) >= 512 && bytes.HasPrefix(raw[257:], []byte{0x75, 0x73, 0x74, 0x61, 0x72}) {
return true
}
if len(raw) < 256 {
return false
}
// The older v7 format has no "magic" field, and therefore must be identified
// with heuristics based on legal ranges of values for other header fields:
// https://www.nationalarchives.gov.uk/PRONOM/Format/proFormatSearch.aspx?status=detailReport&id=385&strPageToDisplay=signatures
rules := []struct {
min, max uint8
i int
}{
{0x21, 0xEF, 0},
{0x30, 0x37, 105},
{0x20, 0x37, 106},
{0x00, 0x00, 107},
{0x30, 0x37, 113},
{0x20, 0x37, 114},
{0x00, 0x00, 115},
{0x30, 0x37, 121},
{0x20, 0x37, 122},
{0x00, 0x00, 123},
{0x30, 0x37, 134},
{0x30, 0x37, 146},
{0x30, 0x37, 153},
{0x00, 0x37, 154},
}
for _, r := range rules {
if raw[r.i] < r.min || raw[r.i] > r.max {
return false
}
}
for _, i := range []uint8{135, 147, 155} {
if raw[i] != 0x00 && raw[i] != 0x20 {
return false
}
}
return true
}
+76
View File
@@ -0,0 +1,76 @@
package magic
import (
"bytes"
"encoding/binary"
)
var (
// Flac matches a Free Lossless Audio Codec file.
Flac = prefix([]byte("\x66\x4C\x61\x43\x00\x00\x00\x22"))
// Midi matches a Musical Instrument Digital Interface file.
Midi = prefix([]byte("\x4D\x54\x68\x64"))
// Ape matches a Monkey's Audio file.
Ape = prefix([]byte("\x4D\x41\x43\x20\x96\x0F\x00\x00\x34\x00\x00\x00\x18\x00\x00\x00\x90\xE3"))
// MusePack matches a Musepack file.
MusePack = prefix([]byte("MPCK"))
// Au matches a Sun Microsystems au file.
Au = prefix([]byte("\x2E\x73\x6E\x64"))
// Amr matches an Adaptive Multi-Rate file.
Amr = prefix([]byte("\x23\x21\x41\x4D\x52"))
// Voc matches a Creative Voice file.
Voc = prefix([]byte("Creative Voice File"))
// M3u matches a Playlist file.
M3u = prefix([]byte("#EXTM3U"))
// AAC matches an Advanced Audio Coding file.
AAC = prefix([]byte{0xFF, 0xF1}, []byte{0xFF, 0xF9})
)
// Mp3 matches an mp3 file.
func Mp3(raw []byte, limit uint32) bool {
if len(raw) < 3 {
return false
}
if bytes.HasPrefix(raw, []byte("ID3")) {
// MP3s with an ID3v2 tag will start with "ID3"
// ID3v1 tags, however appear at the end of the file.
return true
}
// Match MP3 files without tags
switch binary.BigEndian.Uint16(raw[:2]) & 0xFFFE {
case 0xFFFA:
// MPEG ADTS, layer III, v1
return true
case 0xFFF2:
// MPEG ADTS, layer III, v2
return true
case 0xFFE2:
// MPEG ADTS, layer III, v2.5
return true
}
return false
}
// Wav matches a Waveform Audio File Format file.
func Wav(raw []byte, limit uint32) bool {
return len(raw) > 12 &&
bytes.Equal(raw[:4], []byte("RIFF")) &&
bytes.Equal(raw[8:12], []byte{0x57, 0x41, 0x56, 0x45})
}
// Aiff matches Audio Interchange File Format file.
func Aiff(raw []byte, limit uint32) bool {
return len(raw) > 12 &&
bytes.Equal(raw[:4], []byte{0x46, 0x4F, 0x52, 0x4D}) &&
bytes.Equal(raw[8:12], []byte{0x41, 0x49, 0x46, 0x46})
}
// Qcp matches a Qualcomm Pure Voice file.
func Qcp(raw []byte, limit uint32) bool {
return len(raw) > 12 &&
bytes.Equal(raw[:4], []byte("RIFF")) &&
bytes.Equal(raw[8:12], []byte("QLCM"))
}
+198
View File
@@ -0,0 +1,198 @@
package magic
import (
"bytes"
"debug/macho"
"encoding/binary"
)
var (
// Lnk matches Microsoft lnk binary format.
Lnk = prefix([]byte{0x4C, 0x00, 0x00, 0x00, 0x01, 0x14, 0x02, 0x00})
// Wasm matches a web assembly File Format file.
Wasm = prefix([]byte{0x00, 0x61, 0x73, 0x6D})
// Exe matches a Windows/DOS executable file.
Exe = prefix([]byte{0x4D, 0x5A})
// Elf matches an Executable and Linkable Format file.
Elf = prefix([]byte{0x7F, 0x45, 0x4C, 0x46})
// Nes matches a Nintendo Entertainment system ROM file.
Nes = prefix([]byte{0x4E, 0x45, 0x53, 0x1A})
// SWF matches an Adobe Flash swf file.
SWF = prefix([]byte("CWS"), []byte("FWS"), []byte("ZWS"))
// Torrent has bencoded text in the beginning.
Torrent = prefix([]byte("d8:announce"))
)
// Java bytecode and Mach-O binaries share the same magic number.
// More info here https://github.com/threatstack/libmagic/blob/master/magic/Magdir/cafebabe
func classOrMachOFat(in []byte) bool {
// There should be at least 8 bytes for both of them because the only way to
// quickly distinguish them is by comparing byte at position 7
if len(in) < 8 {
return false
}
return bytes.HasPrefix(in, []byte{0xCA, 0xFE, 0xBA, 0xBE})
}
// Class matches a java class file.
func Class(raw []byte, limit uint32) bool {
return classOrMachOFat(raw) && raw[7] > 30
}
// MachO matches Mach-O binaries format.
func MachO(raw []byte, limit uint32) bool {
if classOrMachOFat(raw) && raw[7] < 20 {
return true
}
if len(raw) < 4 {
return false
}
be := binary.BigEndian.Uint32(raw)
le := binary.LittleEndian.Uint32(raw)
return be == macho.Magic32 ||
le == macho.Magic32 ||
be == macho.Magic64 ||
le == macho.Magic64
}
// Dbf matches a dBase file.
// https://www.dbase.com/Knowledgebase/INT/db7_file_fmt.htm
func Dbf(raw []byte, limit uint32) bool {
if len(raw) < 68 {
return false
}
// 3rd and 4th bytes contain the last update month and day of month.
if !(0 < raw[2] && raw[2] < 13 && 0 < raw[3] && raw[3] < 32) {
return false
}
// 12, 13, 30, 31 are reserved bytes and always filled with 0x00.
if raw[12] != 0x00 || raw[13] != 0x00 || raw[30] != 0x00 || raw[31] != 0x00 {
return false
}
// Production MDX flag;
// 0x01 if a production .MDX file exists for this table;
// 0x00 if no .MDX file exists.
if raw[28] > 0x01 {
return false
}
// dbf type is dictated by the first byte.
dbfTypes := []byte{
0x02, 0x03, 0x04, 0x05, 0x30, 0x31, 0x32, 0x42, 0x62, 0x7B, 0x82,
0x83, 0x87, 0x8A, 0x8B, 0x8E, 0xB3, 0xCB, 0xE5, 0xF5, 0xF4, 0xFB,
}
for _, b := range dbfTypes {
if raw[0] == b {
return true
}
}
return false
}
// ElfObj matches an object file.
func ElfObj(raw []byte, limit uint32) bool {
return len(raw) > 17 && ((raw[16] == 0x01 && raw[17] == 0x00) ||
(raw[16] == 0x00 && raw[17] == 0x01))
}
// ElfExe matches an executable file.
func ElfExe(raw []byte, limit uint32) bool {
return len(raw) > 17 && ((raw[16] == 0x02 && raw[17] == 0x00) ||
(raw[16] == 0x00 && raw[17] == 0x02))
}
// ElfLib matches a shared library file.
func ElfLib(raw []byte, limit uint32) bool {
return len(raw) > 17 && ((raw[16] == 0x03 && raw[17] == 0x00) ||
(raw[16] == 0x00 && raw[17] == 0x03))
}
// ElfDump matches a core dump file.
func ElfDump(raw []byte, limit uint32) bool {
return len(raw) > 17 && ((raw[16] == 0x04 && raw[17] == 0x00) ||
(raw[16] == 0x00 && raw[17] == 0x04))
}
// Dcm matches a DICOM medical format file.
func Dcm(raw []byte, limit uint32) bool {
return len(raw) > 131 &&
bytes.Equal(raw[128:132], []byte{0x44, 0x49, 0x43, 0x4D})
}
// Marc matches a MARC21 (MAchine-Readable Cataloging) file.
func Marc(raw []byte, limit uint32) bool {
// File is at least 24 bytes ("leader" field size).
if len(raw) < 24 {
return false
}
// Fixed bytes at offset 20.
if !bytes.Equal(raw[20:24], []byte("4500")) {
return false
}
// First 5 bytes are ASCII digits.
for i := 0; i < 5; i++ {
if raw[i] < '0' || raw[i] > '9' {
return false
}
}
// Field terminator is present in first 2048 bytes.
return bytes.Contains(raw[:min(2048, len(raw))], []byte{0x1E})
}
// Glb matches a glTF model format file.
// GLB is the binary file format representation of 3D models saved in
// the GL transmission Format (glTF).
// GLB uses little endian and its header structure is as follows:
//
// <-- 12-byte header -->
// | magic | version | length |
// | (uint32) | (uint32) | (uint32) |
// | \x67\x6C\x54\x46 | \x01\x00\x00\x00 | ... |
// | g l T F | 1 | ... |
//
// Visit [glTF specification] and [IANA glTF entry] for more details.
//
// [glTF specification]: https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html
// [IANA glTF entry]: https://www.iana.org/assignments/media-types/model/gltf-binary
var Glb = prefix([]byte("\x67\x6C\x54\x46\x02\x00\x00\x00"),
[]byte("\x67\x6C\x54\x46\x01\x00\x00\x00"))
// TzIf matches a Time Zone Information Format (TZif) file.
// See more: https://tools.ietf.org/id/draft-murchison-tzdist-tzif-00.html#rfc.section.3
// Its header structure is shown below:
// +---------------+---+
// | magic (4) | <-+-- version (1)
// +---------------+---+---------------------------------------+
// | [unused - reserved for future use] (15) |
// +---------------+---------------+---------------+-----------+
// | isutccnt (4) | isstdcnt (4) | leapcnt (4) |
// +---------------+---------------+---------------+
// | timecnt (4) | typecnt (4) | charcnt (4) |
func TzIf(raw []byte, limit uint32) bool {
// File is at least 44 bytes (header size).
if len(raw) < 44 {
return false
}
if !bytes.HasPrefix(raw, []byte("TZif")) {
return false
}
// Field "typecnt" MUST not be zero.
if binary.BigEndian.Uint32(raw[36:40]) == 0 {
return false
}
// Version has to be NUL (0x00), '2' (0x32) or '3' (0x33).
return raw[4] == 0x00 || raw[4] == 0x32 || raw[4] == 0x33
}
+13
View File
@@ -0,0 +1,13 @@
package magic
var (
// Sqlite matches an SQLite database file.
Sqlite = prefix([]byte{
0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66,
0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00,
})
// MsAccessAce matches Microsoft Access dababase file.
MsAccessAce = offset([]byte("Standard ACE DB"), 4)
// MsAccessMdb matches legacy Microsoft Access database file (JET, 2003 and earlier).
MsAccessMdb = offset([]byte("Standard Jet DB"), 4)
)
+62
View File
@@ -0,0 +1,62 @@
package magic
import "bytes"
var (
// Pdf matches a Portable Document Format file.
// https://github.com/file/file/blob/11010cc805546a3e35597e67e1129a481aed40e8/magic/Magdir/pdf
Pdf = prefix(
// usual pdf signature
[]byte("%PDF-"),
// new-line prefixed signature
[]byte("\012%PDF-"),
// UTF-8 BOM prefixed signature
[]byte("\xef\xbb\xbf%PDF-"),
)
// Fdf matches a Forms Data Format file.
Fdf = prefix([]byte("%FDF"))
// Mobi matches a Mobi file.
Mobi = offset([]byte("BOOKMOBI"), 60)
// Lit matches a Microsoft Lit file.
Lit = prefix([]byte("ITOLITLS"))
)
// DjVu matches a DjVu file.
func DjVu(raw []byte, limit uint32) bool {
if len(raw) < 12 {
return false
}
if !bytes.HasPrefix(raw, []byte{0x41, 0x54, 0x26, 0x54, 0x46, 0x4F, 0x52, 0x4D}) {
return false
}
return bytes.HasPrefix(raw[12:], []byte("DJVM")) ||
bytes.HasPrefix(raw[12:], []byte("DJVU")) ||
bytes.HasPrefix(raw[12:], []byte("DJVI")) ||
bytes.HasPrefix(raw[12:], []byte("THUM"))
}
// P7s matches an .p7s signature File (PEM, Base64).
func P7s(raw []byte, limit uint32) bool {
// Check for PEM Encoding.
if bytes.HasPrefix(raw, []byte("-----BEGIN PKCS7")) {
return true
}
// Check if DER Encoding is long enough.
if len(raw) < 20 {
return false
}
// Magic Bytes for the signedData ASN.1 encoding.
startHeader := [][]byte{{0x30, 0x80}, {0x30, 0x81}, {0x30, 0x82}, {0x30, 0x83}, {0x30, 0x84}}
signedDataMatch := []byte{0x06, 0x09, 0x2A, 0x86, 0x48, 0x86, 0xF7, 0x0D, 0x01, 0x07}
// Check if Header is correct. There are multiple valid headers.
for i, match := range startHeader {
// If first bytes match, then check for ASN.1 Object Type.
if bytes.HasPrefix(raw, match) {
if bytes.HasPrefix(raw[i+2:], signedDataMatch) {
return true
}
}
}
return false
}
+39
View File
@@ -0,0 +1,39 @@
package magic
import (
"bytes"
)
var (
// Woff matches a Web Open Font Format file.
Woff = prefix([]byte("wOFF"))
// Woff2 matches a Web Open Font Format version 2 file.
Woff2 = prefix([]byte("wOF2"))
// Otf matches an OpenType font file.
Otf = prefix([]byte{0x4F, 0x54, 0x54, 0x4F, 0x00})
)
// Ttf matches a TrueType font file.
func Ttf(raw []byte, limit uint32) bool {
if !bytes.HasPrefix(raw, []byte{0x00, 0x01, 0x00, 0x00}) {
return false
}
return !MsAccessAce(raw, limit) && !MsAccessMdb(raw, limit)
}
// Eot matches an Embedded OpenType font file.
func Eot(raw []byte, limit uint32) bool {
return len(raw) > 35 &&
bytes.Equal(raw[34:36], []byte{0x4C, 0x50}) &&
(bytes.Equal(raw[8:11], []byte{0x02, 0x00, 0x01}) ||
bytes.Equal(raw[8:11], []byte{0x01, 0x00, 0x00}) ||
bytes.Equal(raw[8:11], []byte{0x02, 0x00, 0x02}))
}
// Ttc matches a TrueType Collection font file.
func Ttc(raw []byte, limit uint32) bool {
return len(raw) > 7 &&
bytes.HasPrefix(raw, []byte("ttcf")) &&
(bytes.Equal(raw[4:8], []byte{0x00, 0x01, 0x00, 0x00}) ||
bytes.Equal(raw[4:8], []byte{0x00, 0x02, 0x00, 0x00}))
}
+88
View File
@@ -0,0 +1,88 @@
package magic
import "bytes"
var (
// AVIF matches an AV1 Image File Format still or animated.
// Wikipedia page seems outdated listing image/avif-sequence for animations.
// https://github.com/AOMediaCodec/av1-avif/issues/59
AVIF = ftyp([]byte("avif"), []byte("avis"))
// Mp4 matches an MP4 file.
Mp4 = ftyp(
[]byte("avc1"), []byte("dash"), []byte("iso2"), []byte("iso3"),
[]byte("iso4"), []byte("iso5"), []byte("iso6"), []byte("isom"),
[]byte("mmp4"), []byte("mp41"), []byte("mp42"), []byte("mp4v"),
[]byte("mp71"), []byte("MSNV"), []byte("NDAS"), []byte("NDSC"),
[]byte("NSDC"), []byte("NSDH"), []byte("NDSM"), []byte("NDSP"),
[]byte("NDSS"), []byte("NDXC"), []byte("NDXH"), []byte("NDXM"),
[]byte("NDXP"), []byte("NDXS"), []byte("F4V "), []byte("F4P "),
)
// ThreeGP matches a 3GPP file.
ThreeGP = ftyp(
[]byte("3gp1"), []byte("3gp2"), []byte("3gp3"), []byte("3gp4"),
[]byte("3gp5"), []byte("3gp6"), []byte("3gp7"), []byte("3gs7"),
[]byte("3ge6"), []byte("3ge7"), []byte("3gg6"),
)
// ThreeG2 matches a 3GPP2 file.
ThreeG2 = ftyp(
[]byte("3g24"), []byte("3g25"), []byte("3g26"), []byte("3g2a"),
[]byte("3g2b"), []byte("3g2c"), []byte("KDDI"),
)
// AMp4 matches an audio MP4 file.
AMp4 = ftyp(
// audio for Adobe Flash Player 9+
[]byte("F4A "), []byte("F4B "),
// Apple iTunes AAC-LC (.M4A) Audio
[]byte("M4B "), []byte("M4P "),
// MPEG-4 (.MP4) for SonyPSP
[]byte("MSNV"),
// Nero Digital AAC Audio
[]byte("NDAS"),
)
// Mqv matches a Sony / Mobile QuickTime file.
Mqv = ftyp([]byte("mqt "))
// M4a matches an audio M4A file.
M4a = ftyp([]byte("M4A "))
// M4v matches an Appl4 M4V video file.
M4v = ftyp([]byte("M4V "), []byte("M4VH"), []byte("M4VP"))
// Heic matches a High Efficiency Image Coding (HEIC) file.
Heic = ftyp([]byte("heic"), []byte("heix"))
// HeicSequence matches a High Efficiency Image Coding (HEIC) file sequence.
HeicSequence = ftyp([]byte("hevc"), []byte("hevx"))
// Heif matches a High Efficiency Image File Format (HEIF) file.
Heif = ftyp([]byte("mif1"), []byte("heim"), []byte("heis"), []byte("avic"))
// HeifSequence matches a High Efficiency Image File Format (HEIF) file sequence.
HeifSequence = ftyp([]byte("msf1"), []byte("hevm"), []byte("hevs"), []byte("avcs"))
// TODO: add support for remaining video formats at ftyps.com.
)
// QuickTime matches a QuickTime File Format file.
// https://www.loc.gov/preservation/digital/formats/fdd/fdd000052.shtml
// https://developer.apple.com/library/archive/documentation/QuickTime/QTFF/QTFFChap1/qtff1.html#//apple_ref/doc/uid/TP40000939-CH203-38190
// https://github.com/apache/tika/blob/0f5570691133c75ac4472c3340354a6c4080b104/tika-core/src/main/resources/org/apache/tika/mime/tika-mimetypes.xml#L7758-L7777
func QuickTime(raw []byte, _ uint32) bool {
if len(raw) < 12 {
return false
}
// First 4 bytes represent the size of the atom as unsigned int.
// Next 4 bytes are the type of the atom.
// For `ftyp` atoms check if first byte in size is 0, otherwise, a text file
// which happens to contain 'ftypqt ' at index 4 will trigger a false positive.
if bytes.Equal(raw[4:12], []byte("ftypqt ")) ||
bytes.Equal(raw[4:12], []byte("ftypmoov")) {
return raw[0] == 0x00
}
basicAtomTypes := [][]byte{
[]byte("moov\x00"),
[]byte("mdat\x00"),
[]byte("free\x00"),
[]byte("skip\x00"),
[]byte("pnot\x00"),
}
for _, a := range basicAtomTypes {
if bytes.Equal(raw[4:9], a) {
return true
}
}
return bytes.Equal(raw[:8], []byte("\x00\x00\x00\x08wide"))
}
+55
View File
@@ -0,0 +1,55 @@
package magic
import (
"bytes"
"encoding/binary"
)
// Shp matches a shape format file.
// https://www.esri.com/library/whitepapers/pdfs/shapefile.pdf
func Shp(raw []byte, limit uint32) bool {
if len(raw) < 112 {
return false
}
if !(binary.BigEndian.Uint32(raw[0:4]) == 9994 &&
binary.BigEndian.Uint32(raw[4:8]) == 0 &&
binary.BigEndian.Uint32(raw[8:12]) == 0 &&
binary.BigEndian.Uint32(raw[12:16]) == 0 &&
binary.BigEndian.Uint32(raw[16:20]) == 0 &&
binary.BigEndian.Uint32(raw[20:24]) == 0 &&
binary.LittleEndian.Uint32(raw[28:32]) == 1000) {
return false
}
shapeTypes := []int{
0, // Null shape
1, // Point
3, // Polyline
5, // Polygon
8, // MultiPoint
11, // PointZ
13, // PolylineZ
15, // PolygonZ
18, // MultiPointZ
21, // PointM
23, // PolylineM
25, // PolygonM
28, // MultiPointM
31, // MultiPatch
}
for _, st := range shapeTypes {
if st == int(binary.LittleEndian.Uint32(raw[108:112])) {
return true
}
}
return false
}
// Shx matches a shape index format file.
// https://www.esri.com/library/whitepapers/pdfs/shapefile.pdf
func Shx(raw []byte, limit uint32) bool {
return bytes.HasPrefix(raw, []byte{0x00, 0x00, 0x27, 0x0A})
}
+110
View File
@@ -0,0 +1,110 @@
package magic
import "bytes"
var (
// Png matches a Portable Network Graphics file.
// https://www.w3.org/TR/PNG/
Png = prefix([]byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A})
// Apng matches an Animated Portable Network Graphics file.
// https://wiki.mozilla.org/APNG_Specification
Apng = offset([]byte("acTL"), 37)
// Jpg matches a Joint Photographic Experts Group file.
Jpg = prefix([]byte{0xFF, 0xD8, 0xFF})
// Jp2 matches a JPEG 2000 Image file (ISO 15444-1).
Jp2 = jpeg2k([]byte{0x6a, 0x70, 0x32, 0x20})
// Jpx matches a JPEG 2000 Image file (ISO 15444-2).
Jpx = jpeg2k([]byte{0x6a, 0x70, 0x78, 0x20})
// Jpm matches a JPEG 2000 Image file (ISO 15444-6).
Jpm = jpeg2k([]byte{0x6a, 0x70, 0x6D, 0x20})
// Gif matches a Graphics Interchange Format file.
Gif = prefix([]byte("GIF87a"), []byte("GIF89a"))
// Bmp matches a bitmap image file.
Bmp = prefix([]byte{0x42, 0x4D})
// Ps matches a PostScript file.
Ps = prefix([]byte("%!PS-Adobe-"))
// Psd matches a Photoshop Document file.
Psd = prefix([]byte("8BPS"))
// Ico matches an ICO file.
Ico = prefix([]byte{0x00, 0x00, 0x01, 0x00}, []byte{0x00, 0x00, 0x02, 0x00})
// Icns matches an ICNS (Apple Icon Image format) file.
Icns = prefix([]byte("icns"))
// Tiff matches a Tagged Image File Format file.
Tiff = prefix([]byte{0x49, 0x49, 0x2A, 0x00}, []byte{0x4D, 0x4D, 0x00, 0x2A})
// Bpg matches a Better Portable Graphics file.
Bpg = prefix([]byte{0x42, 0x50, 0x47, 0xFB})
// Xcf matches GIMP image data.
Xcf = prefix([]byte("gimp xcf"))
// Pat matches GIMP pattern data.
Pat = offset([]byte("GPAT"), 20)
// Gbr matches GIMP brush data.
Gbr = offset([]byte("GIMP"), 20)
// Hdr matches Radiance HDR image.
// https://web.archive.org/web/20060913152809/http://local.wasp.uwa.edu.au/~pbourke/dataformats/pic/
Hdr = prefix([]byte("#?RADIANCE\n"))
// Xpm matches X PixMap image data.
Xpm = prefix([]byte{0x2F, 0x2A, 0x20, 0x58, 0x50, 0x4D, 0x20, 0x2A, 0x2F})
// Jxs matches a JPEG XS coded image file (ISO/IEC 21122-3).
Jxs = prefix([]byte{0x00, 0x00, 0x00, 0x0C, 0x4A, 0x58, 0x53, 0x20, 0x0D, 0x0A, 0x87, 0x0A})
// Jxr matches Microsoft HD JXR photo file.
Jxr = prefix([]byte{0x49, 0x49, 0xBC, 0x01})
)
func jpeg2k(sig []byte) Detector {
return func(raw []byte, _ uint32) bool {
if len(raw) < 24 {
return false
}
if !bytes.Equal(raw[4:8], []byte{0x6A, 0x50, 0x20, 0x20}) &&
!bytes.Equal(raw[4:8], []byte{0x6A, 0x50, 0x32, 0x20}) {
return false
}
return bytes.Equal(raw[20:24], sig)
}
}
// Webp matches a WebP file.
func Webp(raw []byte, _ uint32) bool {
return len(raw) > 12 &&
bytes.Equal(raw[0:4], []byte("RIFF")) &&
bytes.Equal(raw[8:12], []byte{0x57, 0x45, 0x42, 0x50})
}
// Dwg matches a CAD drawing file.
func Dwg(raw []byte, _ uint32) bool {
if len(raw) < 6 || raw[0] != 0x41 || raw[1] != 0x43 {
return false
}
dwgVersions := [][]byte{
{0x31, 0x2E, 0x34, 0x30},
{0x31, 0x2E, 0x35, 0x30},
{0x32, 0x2E, 0x31, 0x30},
{0x31, 0x30, 0x30, 0x32},
{0x31, 0x30, 0x30, 0x33},
{0x31, 0x30, 0x30, 0x34},
{0x31, 0x30, 0x30, 0x36},
{0x31, 0x30, 0x30, 0x39},
{0x31, 0x30, 0x31, 0x32},
{0x31, 0x30, 0x31, 0x34},
{0x31, 0x30, 0x31, 0x35},
{0x31, 0x30, 0x31, 0x38},
{0x31, 0x30, 0x32, 0x31},
{0x31, 0x30, 0x32, 0x34},
{0x31, 0x30, 0x33, 0x32},
}
for _, d := range dwgVersions {
if bytes.Equal(raw[2:6], d) {
return true
}
}
return false
}
// Jxl matches JPEG XL image file.
func Jxl(raw []byte, _ uint32) bool {
return bytes.HasPrefix(raw, []byte{0xFF, 0x0A}) ||
bytes.HasPrefix(raw, []byte("\x00\x00\x00\x0cJXL\x20\x0d\x0a\x87\x0a"))
}
+241
View File
@@ -0,0 +1,241 @@
// Package magic holds the matching functions used to find MIME types.
package magic
import (
"bytes"
"fmt"
)
type (
// Detector receiveѕ the raw data of a file and returns whether the data
// meets any conditions. The limit parameter is an upper limit to the number
// of bytes received and is used to tell if the byte slice represents the
// whole file or is just the header of a file: len(raw) < limit or len(raw)>limit.
Detector func(raw []byte, limit uint32) bool
xmlSig struct {
// the local name of the root tag
localName []byte
// the namespace of the XML document
xmlns []byte
}
)
// prefix creates a Detector which returns true if any of the provided signatures
// is the prefix of the raw input.
func prefix(sigs ...[]byte) Detector {
return func(raw []byte, limit uint32) bool {
for _, s := range sigs {
if bytes.HasPrefix(raw, s) {
return true
}
}
return false
}
}
// offset creates a Detector which returns true if the provided signature can be
// found at offset in the raw input.
func offset(sig []byte, offset int) Detector {
return func(raw []byte, limit uint32) bool {
return len(raw) > offset && bytes.HasPrefix(raw[offset:], sig)
}
}
// ciPrefix is like prefix but the check is case insensitive.
func ciPrefix(sigs ...[]byte) Detector {
return func(raw []byte, limit uint32) bool {
for _, s := range sigs {
if ciCheck(s, raw) {
return true
}
}
return false
}
}
func ciCheck(sig, raw []byte) bool {
if len(raw) < len(sig)+1 {
return false
}
// perform case insensitive check
for i, b := range sig {
db := raw[i]
if 'A' <= b && b <= 'Z' {
db &= 0xDF
}
if b != db {
return false
}
}
return true
}
// xml creates a Detector which returns true if any of the provided XML signatures
// matches the raw input.
func xml(sigs ...xmlSig) Detector {
return func(raw []byte, limit uint32) bool {
raw = trimLWS(raw)
if len(raw) == 0 {
return false
}
for _, s := range sigs {
if xmlCheck(s, raw) {
return true
}
}
return false
}
}
func xmlCheck(sig xmlSig, raw []byte) bool {
raw = raw[:min(len(raw), 512)]
if len(sig.localName) == 0 {
return bytes.Index(raw, sig.xmlns) > 0
}
if len(sig.xmlns) == 0 {
return bytes.Index(raw, sig.localName) > 0
}
localNameIndex := bytes.Index(raw, sig.localName)
return localNameIndex != -1 && localNameIndex < bytes.Index(raw, sig.xmlns)
}
// markup creates a Detector which returns true is any of the HTML signatures
// matches the raw input.
func markup(sigs ...[]byte) Detector {
return func(raw []byte, limit uint32) bool {
if bytes.HasPrefix(raw, []byte{0xEF, 0xBB, 0xBF}) {
// We skip the UTF-8 BOM if present to ensure we correctly
// process any leading whitespace. The presence of the BOM
// is taken into account during charset detection in charset.go.
raw = trimLWS(raw[3:])
} else {
raw = trimLWS(raw)
}
if len(raw) == 0 {
return false
}
for _, s := range sigs {
if markupCheck(s, raw) {
return true
}
}
return false
}
}
func markupCheck(sig, raw []byte) bool {
if len(raw) < len(sig)+1 {
return false
}
// perform case insensitive check
for i, b := range sig {
db := raw[i]
if 'A' <= b && b <= 'Z' {
db &= 0xDF
}
if b != db {
return false
}
}
// Next byte must be space or right angle bracket.
if db := raw[len(sig)]; db != ' ' && db != '>' {
return false
}
return true
}
// ftyp creates a Detector which returns true if any of the FTYP signatures
// matches the raw input.
func ftyp(sigs ...[]byte) Detector {
return func(raw []byte, limit uint32) bool {
if len(raw) < 12 {
return false
}
for _, s := range sigs {
if bytes.Equal(raw[4:12], append([]byte("ftyp"), s...)) {
return true
}
}
return false
}
}
func newXMLSig(localName, xmlns string) xmlSig {
ret := xmlSig{xmlns: []byte(xmlns)}
if localName != "" {
ret.localName = []byte(fmt.Sprintf("<%s", localName))
}
return ret
}
// A valid shebang starts with the "#!" characters,
// followed by any number of spaces,
// followed by the path to the interpreter,
// and, optionally, followed by the arguments for the interpreter.
//
// Ex:
//
// #! /usr/bin/env php
//
// /usr/bin/env is the interpreter, php is the first and only argument.
func shebang(sigs ...[]byte) Detector {
return func(raw []byte, limit uint32) bool {
for _, s := range sigs {
if shebangCheck(s, firstLine(raw)) {
return true
}
}
return false
}
}
func shebangCheck(sig, raw []byte) bool {
if len(raw) < len(sig)+2 {
return false
}
if raw[0] != '#' || raw[1] != '!' {
return false
}
return bytes.Equal(trimLWS(trimRWS(raw[2:])), sig)
}
// trimLWS trims whitespace from beginning of the input.
func trimLWS(in []byte) []byte {
firstNonWS := 0
for ; firstNonWS < len(in) && isWS(in[firstNonWS]); firstNonWS++ {
}
return in[firstNonWS:]
}
// trimRWS trims whitespace from the end of the input.
func trimRWS(in []byte) []byte {
lastNonWS := len(in) - 1
for ; lastNonWS > 0 && isWS(in[lastNonWS]); lastNonWS-- {
}
return in[:lastNonWS+1]
}
func firstLine(in []byte) []byte {
lineEnd := 0
for ; lineEnd < len(in) && in[lineEnd] != '\n'; lineEnd++ {
}
return in[:lineEnd]
}
func isWS(b byte) bool {
return b == '\t' || b == '\n' || b == '\x0c' || b == '\r' || b == ' '
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
+225
View File
@@ -0,0 +1,225 @@
package magic
import (
"bytes"
"encoding/binary"
)
var (
xlsxSigFiles = []string{
"xl/worksheets/",
"xl/drawings/",
"xl/theme/",
"xl/_rels/",
"xl/styles.xml",
"xl/workbook.xml",
"xl/sharedStrings.xml",
}
docxSigFiles = []string{
"word/media/",
"word/_rels/document.xml.rels",
"word/document.xml",
"word/styles.xml",
"word/fontTable.xml",
"word/settings.xml",
"word/numbering.xml",
"word/header",
"word/footer",
}
pptxSigFiles = []string{
"ppt/slides/",
"ppt/media/",
"ppt/slideLayouts/",
"ppt/theme/",
"ppt/slideMasters/",
"ppt/tags/",
"ppt/notesMasters/",
"ppt/_rels/",
"ppt/handoutMasters/",
"ppt/notesSlides/",
"ppt/presentation.xml",
"ppt/tableStyles.xml",
"ppt/presProps.xml",
"ppt/viewProps.xml",
}
)
// Xlsx matches a Microsoft Excel 2007 file.
func Xlsx(raw []byte, limit uint32) bool {
return zipContains(raw, xlsxSigFiles...)
}
// Docx matches a Microsoft Word 2007 file.
func Docx(raw []byte, limit uint32) bool {
return zipContains(raw, docxSigFiles...)
}
// Pptx matches a Microsoft PowerPoint 2007 file.
func Pptx(raw []byte, limit uint32) bool {
return zipContains(raw, pptxSigFiles...)
}
// Ole matches an Open Linking and Embedding file.
//
// https://en.wikipedia.org/wiki/Object_Linking_and_Embedding
func Ole(raw []byte, limit uint32) bool {
return bytes.HasPrefix(raw, []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1})
}
// Aaf matches an Advanced Authoring Format file.
// See: https://pyaaf.readthedocs.io/en/latest/about.html
// See: https://en.wikipedia.org/wiki/Advanced_Authoring_Format
func Aaf(raw []byte, limit uint32) bool {
if len(raw) < 31 {
return false
}
return bytes.HasPrefix(raw[8:], []byte{0x41, 0x41, 0x46, 0x42, 0x0D, 0x00, 0x4F, 0x4D}) &&
(raw[30] == 0x09 || raw[30] == 0x0C)
}
// Doc matches a Microsoft Word 97-2003 file.
// See: https://github.com/decalage2/oletools/blob/412ee36ae45e70f42123e835871bac956d958461/oletools/common/clsid.py
func Doc(raw []byte, _ uint32) bool {
clsids := [][]byte{
// Microsoft Word 97-2003 Document (Word.Document.8)
{0x06, 0x09, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46},
// Microsoft Word 6.0-7.0 Document (Word.Document.6)
{0x00, 0x09, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46},
// Microsoft Word Picture (Word.Picture.8)
{0x07, 0x09, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46},
}
for _, clsid := range clsids {
if matchOleClsid(raw, clsid) {
return true
}
}
return false
}
// Ppt matches a Microsoft PowerPoint 97-2003 file or a PowerPoint 95 presentation.
func Ppt(raw []byte, limit uint32) bool {
// Root CLSID test is the safest way to detect identify OLE, however, the format
// often places the root CLSID at the end of the file.
if matchOleClsid(raw, []byte{
0x10, 0x8d, 0x81, 0x64, 0x9b, 0x4f, 0xcf, 0x11,
0x86, 0xea, 0x00, 0xaa, 0x00, 0xb9, 0x29, 0xe8,
}) || matchOleClsid(raw, []byte{
0x70, 0xae, 0x7b, 0xea, 0x3b, 0xfb, 0xcd, 0x11,
0xa9, 0x03, 0x00, 0xaa, 0x00, 0x51, 0x0e, 0xa3,
}) {
return true
}
lin := len(raw)
if lin < 520 {
return false
}
pptSubHeaders := [][]byte{
{0xA0, 0x46, 0x1D, 0xF0},
{0x00, 0x6E, 0x1E, 0xF0},
{0x0F, 0x00, 0xE8, 0x03},
}
for _, h := range pptSubHeaders {
if bytes.HasPrefix(raw[512:], h) {
return true
}
}
if bytes.HasPrefix(raw[512:], []byte{0xFD, 0xFF, 0xFF, 0xFF}) &&
raw[518] == 0x00 && raw[519] == 0x00 {
return true
}
return lin > 1152 && bytes.Contains(raw[1152:min(4096, lin)],
[]byte("P\x00o\x00w\x00e\x00r\x00P\x00o\x00i\x00n\x00t\x00 D\x00o\x00c\x00u\x00m\x00e\x00n\x00t"))
}
// Xls matches a Microsoft Excel 97-2003 file.
func Xls(raw []byte, limit uint32) bool {
// Root CLSID test is the safest way to detect identify OLE, however, the format
// often places the root CLSID at the end of the file.
if matchOleClsid(raw, []byte{
0x10, 0x08, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,
}) || matchOleClsid(raw, []byte{
0x20, 0x08, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,
}) {
return true
}
lin := len(raw)
if lin < 520 {
return false
}
xlsSubHeaders := [][]byte{
{0x09, 0x08, 0x10, 0x00, 0x00, 0x06, 0x05, 0x00},
{0xFD, 0xFF, 0xFF, 0xFF, 0x10},
{0xFD, 0xFF, 0xFF, 0xFF, 0x1F},
{0xFD, 0xFF, 0xFF, 0xFF, 0x22},
{0xFD, 0xFF, 0xFF, 0xFF, 0x23},
{0xFD, 0xFF, 0xFF, 0xFF, 0x28},
{0xFD, 0xFF, 0xFF, 0xFF, 0x29},
}
for _, h := range xlsSubHeaders {
if bytes.HasPrefix(raw[512:], h) {
return true
}
}
return lin > 1152 && bytes.Contains(raw[1152:min(4096, lin)],
[]byte("W\x00k\x00s\x00S\x00S\x00W\x00o\x00r\x00k\x00B\x00o\x00o\x00k"))
}
// Pub matches a Microsoft Publisher file.
func Pub(raw []byte, limit uint32) bool {
return matchOleClsid(raw, []byte{
0x01, 0x12, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46,
})
}
// Msg matches a Microsoft Outlook email file.
func Msg(raw []byte, limit uint32) bool {
return matchOleClsid(raw, []byte{
0x0B, 0x0D, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,
0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46,
})
}
// Msi matches a Microsoft Windows Installer file.
// http://fileformats.archiveteam.org/wiki/Microsoft_Compound_File
func Msi(raw []byte, limit uint32) bool {
return matchOleClsid(raw, []byte{
0x84, 0x10, 0x0C, 0x00, 0x00, 0x00, 0x00, 0x00,
0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46,
})
}
// Helper to match by a specific CLSID of a compound file.
//
// http://fileformats.archiveteam.org/wiki/Microsoft_Compound_File
func matchOleClsid(in []byte, clsid []byte) bool {
// Microsoft Compound files v3 have a sector length of 512, while v4 has 4096.
// Change sector offset depending on file version.
// https://www.loc.gov/preservation/digital/formats/fdd/fdd000392.shtml
sectorLength := 512
if len(in) < sectorLength {
return false
}
if in[26] == 0x04 && in[27] == 0x00 {
sectorLength = 4096
}
// SecID of first sector of the directory stream.
firstSecID := int(binary.LittleEndian.Uint32(in[48:52]))
// Expected offset of CLSID for root storage object.
clsidOffset := sectorLength*(1+firstSecID) + 80
if len(in) <= clsidOffset+16 {
return false
}
return bytes.HasPrefix(in[clsidOffset:], clsid)
}
+42
View File
@@ -0,0 +1,42 @@
package magic
import (
"bytes"
)
/*
NOTE:
In May 2003, two Internet RFCs were published relating to the format.
The Ogg bitstream was defined in RFC 3533 (which is classified as
'informative') and its Internet content type (application/ogg) in RFC
3534 (which is, as of 2006, a proposed standard protocol). In
September 2008, RFC 3534 was obsoleted by RFC 5334, which added
content types video/ogg, audio/ogg and filename extensions .ogx, .ogv,
.oga, .spx.
See:
https://tools.ietf.org/html/rfc3533
https://developer.mozilla.org/en-US/docs/Web/HTTP/Configuring_servers_for_Ogg_media#Serve_media_with_the_correct_MIME_type
https://github.com/file/file/blob/master/magic/Magdir/vorbis
*/
// Ogg matches an Ogg file.
func Ogg(raw []byte, limit uint32) bool {
return bytes.HasPrefix(raw, []byte("\x4F\x67\x67\x53\x00"))
}
// OggAudio matches an audio ogg file.
func OggAudio(raw []byte, limit uint32) bool {
return len(raw) >= 37 && (bytes.HasPrefix(raw[28:], []byte("\x7fFLAC")) ||
bytes.HasPrefix(raw[28:], []byte("\x01vorbis")) ||
bytes.HasPrefix(raw[28:], []byte("OpusHead")) ||
bytes.HasPrefix(raw[28:], []byte("Speex\x20\x20\x20")))
}
// OggVideo matches a video ogg file.
func OggVideo(raw []byte, limit uint32) bool {
return len(raw) >= 37 && (bytes.HasPrefix(raw[28:], []byte("\x80theora")) ||
bytes.HasPrefix(raw[28:], []byte("fishead\x00")) ||
bytes.HasPrefix(raw[28:], []byte("\x01video\x00\x00\x00"))) // OGM video
}
+375
View File
@@ -0,0 +1,375 @@
package magic
import (
"bufio"
"bytes"
"strings"
"time"
"github.com/gabriel-vasile/mimetype/internal/charset"
"github.com/gabriel-vasile/mimetype/internal/json"
)
var (
// HTML matches a Hypertext Markup Language file.
HTML = markup(
[]byte("<!DOCTYPE HTML"),
[]byte("<HTML"),
[]byte("<HEAD"),
[]byte("<SCRIPT"),
[]byte("<IFRAME"),
[]byte("<H1"),
[]byte("<DIV"),
[]byte("<FONT"),
[]byte("<TABLE"),
[]byte("<A"),
[]byte("<STYLE"),
[]byte("<TITLE"),
[]byte("<B"),
[]byte("<BODY"),
[]byte("<BR"),
[]byte("<P"),
)
// XML matches an Extensible Markup Language file.
XML = markup([]byte("<?XML"))
// Owl2 matches an Owl ontology file.
Owl2 = xml(newXMLSig("Ontology", `xmlns="http://www.w3.org/2002/07/owl#"`))
// Rss matches a Rich Site Summary file.
Rss = xml(newXMLSig("rss", ""))
// Atom matches an Atom Syndication Format file.
Atom = xml(newXMLSig("feed", `xmlns="http://www.w3.org/2005/Atom"`))
// Kml matches a Keyhole Markup Language file.
Kml = xml(
newXMLSig("kml", `xmlns="http://www.opengis.net/kml/2.2"`),
newXMLSig("kml", `xmlns="http://earth.google.com/kml/2.0"`),
newXMLSig("kml", `xmlns="http://earth.google.com/kml/2.1"`),
newXMLSig("kml", `xmlns="http://earth.google.com/kml/2.2"`),
)
// Xliff matches a XML Localization Interchange File Format file.
Xliff = xml(newXMLSig("xliff", `xmlns="urn:oasis:names:tc:xliff:document:1.2"`))
// Collada matches a COLLAborative Design Activity file.
Collada = xml(newXMLSig("COLLADA", `xmlns="http://www.collada.org/2005/11/COLLADASchema"`))
// Gml matches a Geography Markup Language file.
Gml = xml(
newXMLSig("", `xmlns:gml="http://www.opengis.net/gml"`),
newXMLSig("", `xmlns:gml="http://www.opengis.net/gml/3.2"`),
newXMLSig("", `xmlns:gml="http://www.opengis.net/gml/3.3/exr"`),
)
// Gpx matches a GPS Exchange Format file.
Gpx = xml(newXMLSig("gpx", `xmlns="http://www.topografix.com/GPX/1/1"`))
// Tcx matches a Training Center XML file.
Tcx = xml(newXMLSig("TrainingCenterDatabase", `xmlns="http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2"`))
// X3d matches an Extensible 3D Graphics file.
X3d = xml(newXMLSig("X3D", `xmlns:xsd="http://www.w3.org/2001/XMLSchema-instance"`))
// Amf matches an Additive Manufacturing XML file.
Amf = xml(newXMLSig("amf", ""))
// Threemf matches a 3D Manufacturing Format file.
Threemf = xml(newXMLSig("model", `xmlns="http://schemas.microsoft.com/3dmanufacturing/core/2015/02"`))
// Xfdf matches a XML Forms Data Format file.
Xfdf = xml(newXMLSig("xfdf", `xmlns="http://ns.adobe.com/xfdf/"`))
// VCard matches a Virtual Contact File.
VCard = ciPrefix([]byte("BEGIN:VCARD\n"), []byte("BEGIN:VCARD\r\n"))
// ICalendar matches a iCalendar file.
ICalendar = ciPrefix([]byte("BEGIN:VCALENDAR\n"), []byte("BEGIN:VCALENDAR\r\n"))
phpPageF = ciPrefix(
[]byte("<?PHP"),
[]byte("<?\n"),
[]byte("<?\r"),
[]byte("<? "),
)
phpScriptF = shebang(
[]byte("/usr/local/bin/php"),
[]byte("/usr/bin/php"),
[]byte("/usr/bin/env php"),
)
// Js matches a Javascript file.
Js = shebang(
[]byte("/bin/node"),
[]byte("/usr/bin/node"),
[]byte("/bin/nodejs"),
[]byte("/usr/bin/nodejs"),
[]byte("/usr/bin/env node"),
[]byte("/usr/bin/env nodejs"),
)
// Lua matches a Lua programming language file.
Lua = shebang(
[]byte("/usr/bin/lua"),
[]byte("/usr/local/bin/lua"),
[]byte("/usr/bin/env lua"),
)
// Perl matches a Perl programming language file.
Perl = shebang(
[]byte("/usr/bin/perl"),
[]byte("/usr/bin/env perl"),
)
// Python matches a Python programming language file.
Python = shebang(
[]byte("/usr/bin/python"),
[]byte("/usr/local/bin/python"),
[]byte("/usr/bin/env python"),
)
// Tcl matches a Tcl programming language file.
Tcl = shebang(
[]byte("/usr/bin/tcl"),
[]byte("/usr/local/bin/tcl"),
[]byte("/usr/bin/env tcl"),
[]byte("/usr/bin/tclsh"),
[]byte("/usr/local/bin/tclsh"),
[]byte("/usr/bin/env tclsh"),
[]byte("/usr/bin/wish"),
[]byte("/usr/local/bin/wish"),
[]byte("/usr/bin/env wish"),
)
// Rtf matches a Rich Text Format file.
Rtf = prefix([]byte("{\\rtf1"))
)
// Text matches a plain text file.
//
// TODO: This function does not parse BOM-less UTF16 and UTF32 files. Not really
// sure it should. Linux file utility also requires a BOM for UTF16 and UTF32.
func Text(raw []byte, limit uint32) bool {
// First look for BOM.
if cset := charset.FromBOM(raw); cset != "" {
return true
}
// Binary data bytes as defined here: https://mimesniff.spec.whatwg.org/#binary-data-byte
for _, b := range raw {
if b <= 0x08 ||
b == 0x0B ||
0x0E <= b && b <= 0x1A ||
0x1C <= b && b <= 0x1F {
return false
}
}
return true
}
// Php matches a PHP: Hypertext Preprocessor file.
func Php(raw []byte, limit uint32) bool {
if res := phpPageF(raw, limit); res {
return res
}
return phpScriptF(raw, limit)
}
// JSON matches a JavaScript Object Notation file.
func JSON(raw []byte, limit uint32) bool {
raw = trimLWS(raw)
// #175 A single JSON string, number or bool is not considered JSON.
// JSON objects and arrays are reported as JSON.
if len(raw) < 2 || (raw[0] != '[' && raw[0] != '{') {
return false
}
parsed, err := json.Scan(raw)
// If the full file content was provided, check there is no error.
if limit == 0 || len(raw) < int(limit) {
return err == nil
}
// If a section of the file was provided, check if all of it was parsed.
return parsed == len(raw) && len(raw) > 0
}
// GeoJSON matches a RFC 7946 GeoJSON file.
//
// GeoJSON detection implies searching for key:value pairs like: `"type": "Feature"`
// in the input.
// BUG(gabriel-vasile): The "type" key should be searched for in the root object.
func GeoJSON(raw []byte, limit uint32) bool {
raw = trimLWS(raw)
if len(raw) == 0 {
return false
}
// GeoJSON is always a JSON object, not a JSON array or any other JSON value.
if raw[0] != '{' {
return false
}
s := []byte(`"type"`)
si, sl := bytes.Index(raw, s), len(s)
if si == -1 {
return false
}
// If the "type" string is the suffix of the input,
// there is no need to search for the value of the key.
if si+sl == len(raw) {
return false
}
// Skip the "type" part.
raw = raw[si+sl:]
// Skip any whitespace before the colon.
raw = trimLWS(raw)
// Check for colon.
if len(raw) == 0 || raw[0] != ':' {
return false
}
// Skip any whitespace after the colon.
raw = trimLWS(raw[1:])
geoJSONTypes := [][]byte{
[]byte(`"Feature"`),
[]byte(`"FeatureCollection"`),
[]byte(`"Point"`),
[]byte(`"LineString"`),
[]byte(`"Polygon"`),
[]byte(`"MultiPoint"`),
[]byte(`"MultiLineString"`),
[]byte(`"MultiPolygon"`),
[]byte(`"GeometryCollection"`),
}
for _, t := range geoJSONTypes {
if bytes.HasPrefix(raw, t) {
return true
}
}
return false
}
// NdJSON matches a Newline delimited JSON file. All complete lines from raw
// must be valid JSON documents meaning they contain one of the valid JSON data
// types.
func NdJSON(raw []byte, limit uint32) bool {
lCount, hasObjOrArr := 0, false
sc := bufio.NewScanner(dropLastLine(raw, limit))
for sc.Scan() {
l := sc.Bytes()
// Empty lines are allowed in NDJSON.
if l = trimRWS(trimLWS(l)); len(l) == 0 {
continue
}
_, err := json.Scan(l)
if err != nil {
return false
}
if l[0] == '[' || l[0] == '{' {
hasObjOrArr = true
}
lCount++
}
return lCount > 1 && hasObjOrArr
}
// HAR matches a HAR Spec file.
// Spec: http://www.softwareishard.com/blog/har-12-spec/
func HAR(raw []byte, limit uint32) bool {
s := []byte(`"log"`)
si, sl := bytes.Index(raw, s), len(s)
if si == -1 {
return false
}
// If the "log" string is the suffix of the input,
// there is no need to search for the value of the key.
if si+sl == len(raw) {
return false
}
// Skip the "log" part.
raw = raw[si+sl:]
// Skip any whitespace before the colon.
raw = trimLWS(raw)
// Check for colon.
if len(raw) == 0 || raw[0] != ':' {
return false
}
// Skip any whitespace after the colon.
raw = trimLWS(raw[1:])
harJSONTypes := [][]byte{
[]byte(`"version"`),
[]byte(`"creator"`),
[]byte(`"entries"`),
}
for _, t := range harJSONTypes {
si := bytes.Index(raw, t)
if si > -1 {
return true
}
}
return false
}
// Svg matches a SVG file.
func Svg(raw []byte, limit uint32) bool {
return bytes.Contains(raw, []byte("<svg"))
}
// Srt matches a SubRip file.
func Srt(in []byte, _ uint32) bool {
s := bufio.NewScanner(bytes.NewReader(in))
if !s.Scan() {
return false
}
// First line must be 1.
if s.Text() != "1" {
return false
}
if !s.Scan() {
return false
}
secondLine := s.Text()
// Timestamp format (e.g: 00:02:16,612 --> 00:02:19,376) limits secondLine
// length to exactly 29 characters.
if len(secondLine) != 29 {
return false
}
// Decimal separator of fractional seconds in the timestamps must be a
// comma, not a period.
if strings.Contains(secondLine, ".") {
return false
}
// For Go <1.17, comma is not recognised as a decimal separator by `time.Parse`.
secondLine = strings.ReplaceAll(secondLine, ",", ".")
// Second line must be a time range.
ts := strings.Split(secondLine, " --> ")
if len(ts) != 2 {
return false
}
const layout = "15:04:05.000"
t0, err := time.Parse(layout, ts[0])
if err != nil {
return false
}
t1, err := time.Parse(layout, ts[1])
if err != nil {
return false
}
if t0.After(t1) {
return false
}
// A third line must exist and not be empty. This is the actual subtitle text.
return s.Scan() && len(s.Bytes()) != 0
}
// Vtt matches a Web Video Text Tracks (WebVTT) file. See
// https://www.iana.org/assignments/media-types/text/vtt.
func Vtt(raw []byte, limit uint32) bool {
// Prefix match.
prefixes := [][]byte{
{0xEF, 0xBB, 0xBF, 0x57, 0x45, 0x42, 0x56, 0x54, 0x54, 0x0A}, // UTF-8 BOM, "WEBVTT" and a line feed
{0xEF, 0xBB, 0xBF, 0x57, 0x45, 0x42, 0x56, 0x54, 0x54, 0x0D}, // UTF-8 BOM, "WEBVTT" and a carriage return
{0xEF, 0xBB, 0xBF, 0x57, 0x45, 0x42, 0x56, 0x54, 0x54, 0x20}, // UTF-8 BOM, "WEBVTT" and a space
{0xEF, 0xBB, 0xBF, 0x57, 0x45, 0x42, 0x56, 0x54, 0x54, 0x09}, // UTF-8 BOM, "WEBVTT" and a horizontal tab
{0x57, 0x45, 0x42, 0x56, 0x54, 0x54, 0x0A}, // "WEBVTT" and a line feed
{0x57, 0x45, 0x42, 0x56, 0x54, 0x54, 0x0D}, // "WEBVTT" and a carriage return
{0x57, 0x45, 0x42, 0x56, 0x54, 0x54, 0x20}, // "WEBVTT" and a space
{0x57, 0x45, 0x42, 0x56, 0x54, 0x54, 0x09}, // "WEBVTT" and a horizontal tab
}
for _, p := range prefixes {
if bytes.HasPrefix(raw, p) {
return true
}
}
// Exact match.
return bytes.Equal(raw, []byte{0xEF, 0xBB, 0xBF, 0x57, 0x45, 0x42, 0x56, 0x54, 0x54}) || // UTF-8 BOM and "WEBVTT"
bytes.Equal(raw, []byte{0x57, 0x45, 0x42, 0x56, 0x54, 0x54}) // "WEBVTT"
}
+63
View File
@@ -0,0 +1,63 @@
package magic
import (
"bytes"
"encoding/csv"
"errors"
"io"
)
// Csv matches a comma-separated values file.
func Csv(raw []byte, limit uint32) bool {
return sv(raw, ',', limit)
}
// Tsv matches a tab-separated values file.
func Tsv(raw []byte, limit uint32) bool {
return sv(raw, '\t', limit)
}
func sv(in []byte, comma rune, limit uint32) bool {
r := csv.NewReader(dropLastLine(in, limit))
r.Comma = comma
r.ReuseRecord = true
r.LazyQuotes = true
r.Comment = '#'
lines := 0
for {
_, err := r.Read()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return false
}
lines++
}
return r.FieldsPerRecord > 1 && lines > 1
}
// dropLastLine drops the last incomplete line from b.
//
// mimetype limits itself to ReadLimit bytes when performing a detection.
// This means, for file formats like CSV for NDJSON, the last line of the input
// can be an incomplete line.
func dropLastLine(b []byte, cutAt uint32) io.Reader {
if cutAt == 0 {
return bytes.NewReader(b)
}
if uint32(len(b)) >= cutAt {
for i := cutAt - 1; i > 0; i-- {
if b[i] == '\n' {
return bytes.NewReader(b[:i])
}
}
// No newline was found between the 0 index and cutAt.
return bytes.NewReader(b[:cutAt])
}
return bytes.NewReader(b)
}
+85
View File
@@ -0,0 +1,85 @@
package magic
import (
"bytes"
)
var (
// Flv matches a Flash video file.
Flv = prefix([]byte("\x46\x4C\x56\x01"))
// Asf matches an Advanced Systems Format file.
Asf = prefix([]byte{
0x30, 0x26, 0xB2, 0x75, 0x8E, 0x66, 0xCF, 0x11,
0xA6, 0xD9, 0x00, 0xAA, 0x00, 0x62, 0xCE, 0x6C,
})
// Rmvb matches a RealMedia Variable Bitrate file.
Rmvb = prefix([]byte{0x2E, 0x52, 0x4D, 0x46})
)
// WebM matches a WebM file.
func WebM(raw []byte, limit uint32) bool {
return isMatroskaFileTypeMatched(raw, "webm")
}
// Mkv matches a mkv file.
func Mkv(raw []byte, limit uint32) bool {
return isMatroskaFileTypeMatched(raw, "matroska")
}
// isMatroskaFileTypeMatched is used for webm and mkv file matching.
// It checks for .Eߣ sequence. If the sequence is found,
// then it means it is Matroska media container, including WebM.
// Then it verifies which of the file type it is representing by matching the
// file specific string.
func isMatroskaFileTypeMatched(in []byte, flType string) bool {
if bytes.HasPrefix(in, []byte("\x1A\x45\xDF\xA3")) {
return isFileTypeNamePresent(in, flType)
}
return false
}
// isFileTypeNamePresent accepts the matroska input data stream and searches
// for the given file type in the stream. Return whether a match is found.
// The logic of search is: find first instance of \x42\x82 and then
// search for given string after n bytes of above instance.
func isFileTypeNamePresent(in []byte, flType string) bool {
ind, maxInd, lenIn := 0, 4096, len(in)
if lenIn < maxInd { // restricting length to 4096
maxInd = lenIn
}
ind = bytes.Index(in[:maxInd], []byte("\x42\x82"))
if ind > 0 && lenIn > ind+2 {
ind += 2
// filetype name will be present exactly
// n bytes after the match of the two bytes "\x42\x82"
n := vintWidth(int(in[ind]))
if lenIn > ind+n {
return bytes.HasPrefix(in[ind+n:], []byte(flType))
}
}
return false
}
// vintWidth parses the variable-integer width in matroska containers
func vintWidth(v int) int {
mask, max, num := 128, 8, 1
for num < max && v&mask == 0 {
mask = mask >> 1
num++
}
return num
}
// Mpeg matches a Moving Picture Experts Group file.
func Mpeg(raw []byte, limit uint32) bool {
return len(raw) > 3 && bytes.HasPrefix(raw, []byte{0x00, 0x00, 0x01}) &&
raw[3] >= 0xB0 && raw[3] <= 0xBF
}
// Avi matches an Audio Video Interleaved file.
func Avi(raw []byte, limit uint32) bool {
return len(raw) > 16 &&
bytes.Equal(raw[:4], []byte("RIFF")) &&
bytes.Equal(raw[8:16], []byte("AVI LIST"))
}
+92
View File
@@ -0,0 +1,92 @@
package magic
import (
"bytes"
"encoding/binary"
"strings"
)
var (
// Odt matches an OpenDocument Text file.
Odt = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.text"), 30)
// Ott matches an OpenDocument Text Template file.
Ott = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.text-template"), 30)
// Ods matches an OpenDocument Spreadsheet file.
Ods = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.spreadsheet"), 30)
// Ots matches an OpenDocument Spreadsheet Template file.
Ots = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.spreadsheet-template"), 30)
// Odp matches an OpenDocument Presentation file.
Odp = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.presentation"), 30)
// Otp matches an OpenDocument Presentation Template file.
Otp = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.presentation-template"), 30)
// Odg matches an OpenDocument Drawing file.
Odg = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.graphics"), 30)
// Otg matches an OpenDocument Drawing Template file.
Otg = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.graphics-template"), 30)
// Odf matches an OpenDocument Formula file.
Odf = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.formula"), 30)
// Odc matches an OpenDocument Chart file.
Odc = offset([]byte("mimetypeapplication/vnd.oasis.opendocument.chart"), 30)
// Epub matches an EPUB file.
Epub = offset([]byte("mimetypeapplication/epub+zip"), 30)
// Sxc matches an OpenOffice Spreadsheet file.
Sxc = offset([]byte("mimetypeapplication/vnd.sun.xml.calc"), 30)
)
// Zip matches a zip archive.
func Zip(raw []byte, limit uint32) bool {
return len(raw) > 3 &&
raw[0] == 0x50 && raw[1] == 0x4B &&
(raw[2] == 0x3 || raw[2] == 0x5 || raw[2] == 0x7) &&
(raw[3] == 0x4 || raw[3] == 0x6 || raw[3] == 0x8)
}
// Jar matches a Java archive file.
func Jar(raw []byte, limit uint32) bool {
return zipContains(raw, "META-INF/MANIFEST.MF")
}
// zipTokenizer holds the source zip file and scanned index.
type zipTokenizer struct {
in []byte
i int // current index
}
// next returns the next file name from the zip headers.
// https://web.archive.org/web/20191129114319/https://users.cs.jmu.edu/buchhofp/forensics/formats/pkzip.html
func (t *zipTokenizer) next() (fileName string) {
if t.i > len(t.in) {
return
}
in := t.in[t.i:]
// pkSig is the signature of the zip local file header.
pkSig := []byte("PK\003\004")
pkIndex := bytes.Index(in, pkSig)
// 30 is the offset of the file name in the header.
fNameOffset := pkIndex + 30
// end if signature not found or file name offset outside of file.
if pkIndex == -1 || fNameOffset > len(in) {
return
}
fNameLen := int(binary.LittleEndian.Uint16(in[pkIndex+26 : pkIndex+28]))
if fNameLen <= 0 || fNameOffset+fNameLen > len(in) {
return
}
t.i += fNameOffset + fNameLen
return string(in[fNameOffset : fNameOffset+fNameLen])
}
// zipContains returns true if the zip file headers from in contain any of the paths.
func zipContains(in []byte, paths ...string) bool {
t := zipTokenizer{in: in}
for i, tok := 0, t.next(); tok != ""; i, tok = i+1, t.next() {
for p := range paths {
if strings.HasPrefix(tok, paths[p]) {
return true
}
}
}
return false
}
+186
View File
@@ -0,0 +1,186 @@
package mimetype
import (
"mime"
"github.com/gabriel-vasile/mimetype/internal/charset"
"github.com/gabriel-vasile/mimetype/internal/magic"
)
// MIME struct holds information about a file format: the string representation
// of the MIME type, the extension and the parent file format.
type MIME struct {
mime string
aliases []string
extension string
// detector receives the raw input and a limit for the number of bytes it is
// allowed to check. It returns whether the input matches a signature or not.
detector magic.Detector
children []*MIME
parent *MIME
}
// String returns the string representation of the MIME type, e.g., "application/zip".
func (m *MIME) String() string {
return m.mime
}
// Extension returns the file extension associated with the MIME type.
// It includes the leading dot, as in ".html". When the file format does not
// have an extension, the empty string is returned.
func (m *MIME) Extension() string {
return m.extension
}
// Parent returns the parent MIME type from the hierarchy.
// Each MIME type has a non-nil parent, except for the root MIME type.
//
// For example, the application/json and text/html MIME types have text/plain as
// their parent because they are text files who happen to contain JSON or HTML.
// Another example is the ZIP format, which is used as container
// for Microsoft Office files, EPUB files, JAR files, and others.
func (m *MIME) Parent() *MIME {
return m.parent
}
// Is checks whether this MIME type, or any of its aliases, is equal to the
// expected MIME type. MIME type equality test is done on the "type/subtype"
// section, ignores any optional MIME parameters, ignores any leading and
// trailing whitespace, and is case insensitive.
func (m *MIME) Is(expectedMIME string) bool {
// Parsing is needed because some detected MIME types contain parameters
// that need to be stripped for the comparison.
expectedMIME, _, _ = mime.ParseMediaType(expectedMIME)
found, _, _ := mime.ParseMediaType(m.mime)
if expectedMIME == found {
return true
}
for _, alias := range m.aliases {
if alias == expectedMIME {
return true
}
}
return false
}
func newMIME(
mime, extension string,
detector magic.Detector,
children ...*MIME) *MIME {
m := &MIME{
mime: mime,
extension: extension,
detector: detector,
children: children,
}
for _, c := range children {
c.parent = m
}
return m
}
func (m *MIME) alias(aliases ...string) *MIME {
m.aliases = aliases
return m
}
// match does a depth-first search on the signature tree. It returns the deepest
// successful node for which all the children detection functions fail.
func (m *MIME) match(in []byte, readLimit uint32) *MIME {
for _, c := range m.children {
if c.detector(in, readLimit) {
return c.match(in, readLimit)
}
}
needsCharset := map[string]func([]byte) string{
"text/plain": charset.FromPlain,
"text/html": charset.FromHTML,
"text/xml": charset.FromXML,
}
// ps holds optional MIME parameters.
ps := map[string]string{}
if f, ok := needsCharset[m.mime]; ok {
if cset := f(in); cset != "" {
ps["charset"] = cset
}
}
return m.cloneHierarchy(ps)
}
// flatten transforms an hierarchy of MIMEs into a slice of MIMEs.
func (m *MIME) flatten() []*MIME {
out := []*MIME{m}
for _, c := range m.children {
out = append(out, c.flatten()...)
}
return out
}
// clone creates a new MIME with the provided optional MIME parameters.
func (m *MIME) clone(ps map[string]string) *MIME {
clonedMIME := m.mime
if len(ps) > 0 {
clonedMIME = mime.FormatMediaType(m.mime, ps)
}
return &MIME{
mime: clonedMIME,
aliases: m.aliases,
extension: m.extension,
}
}
// cloneHierarchy creates a clone of m and all its ancestors. The optional MIME
// parameters are set on the last child of the hierarchy.
func (m *MIME) cloneHierarchy(ps map[string]string) *MIME {
ret := m.clone(ps)
lastChild := ret
for p := m.Parent(); p != nil; p = p.Parent() {
pClone := p.clone(nil)
lastChild.parent = pClone
lastChild = pClone
}
return ret
}
func (m *MIME) lookup(mime string) *MIME {
for _, n := range append(m.aliases, m.mime) {
if n == mime {
return m
}
}
for _, c := range m.children {
if m := c.lookup(mime); m != nil {
return m
}
}
return nil
}
// Extend adds detection for a sub-format. The detector is a function
// returning true when the raw input file satisfies a signature.
// The sub-format will be detected if all the detectors in the parent chain return true.
// The extension should include the leading dot, as in ".html".
func (m *MIME) Extend(detector func(raw []byte, limit uint32) bool, mime, extension string, aliases ...string) {
c := &MIME{
mime: mime,
extension: extension,
detector: detector,
parent: m,
aliases: aliases,
}
mu.Lock()
m.children = append([]*MIME{c}, m.children...)
mu.Unlock()
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 MiB

+124
View File
@@ -0,0 +1,124 @@
// Package mimetype uses magic number signatures to detect the MIME type of a file.
//
// File formats are stored in a hierarchy with application/octet-stream at its root.
// For example, the hierarchy for HTML format is application/octet-stream ->
// text/plain -> text/html.
package mimetype
import (
"io"
"io/ioutil"
"mime"
"os"
"sync/atomic"
)
// readLimit is the maximum number of bytes from the input used when detecting.
var readLimit uint32 = 3072
// Detect returns the MIME type found from the provided byte slice.
//
// The result is always a valid MIME type, with application/octet-stream
// returned when identification failed.
func Detect(in []byte) *MIME {
// Using atomic because readLimit can be written at the same time in other goroutine.
l := atomic.LoadUint32(&readLimit)
if l > 0 && len(in) > int(l) {
in = in[:l]
}
mu.RLock()
defer mu.RUnlock()
return root.match(in, l)
}
// DetectReader returns the MIME type of the provided reader.
//
// The result is always a valid MIME type, with application/octet-stream
// returned when identification failed with or without an error.
// Any error returned is related to the reading from the input reader.
//
// DetectReader assumes the reader offset is at the start. If the input is an
// io.ReadSeeker you previously read from, it should be rewinded before detection:
//
// reader.Seek(0, io.SeekStart)
func DetectReader(r io.Reader) (*MIME, error) {
var in []byte
var err error
// Using atomic because readLimit can be written at the same time in other goroutine.
l := atomic.LoadUint32(&readLimit)
if l == 0 {
in, err = ioutil.ReadAll(r)
if err != nil {
return errMIME, err
}
} else {
var n int
in = make([]byte, l)
// io.UnexpectedEOF means len(r) < len(in). It is not an error in this case,
// it just means the input file is smaller than the allocated bytes slice.
n, err = io.ReadFull(r, in)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
return errMIME, err
}
in = in[:n]
}
mu.RLock()
defer mu.RUnlock()
return root.match(in, l), nil
}
// DetectFile returns the MIME type of the provided file.
//
// The result is always a valid MIME type, with application/octet-stream
// returned when identification failed with or without an error.
// Any error returned is related to the opening and reading from the input file.
func DetectFile(path string) (*MIME, error) {
f, err := os.Open(path)
if err != nil {
return errMIME, err
}
defer f.Close()
return DetectReader(f)
}
// EqualsAny reports whether s MIME type is equal to any MIME type in mimes.
// MIME type equality test is done on the "type/subtype" section, ignores
// any optional MIME parameters, ignores any leading and trailing whitespace,
// and is case insensitive.
func EqualsAny(s string, mimes ...string) bool {
s, _, _ = mime.ParseMediaType(s)
for _, m := range mimes {
m, _, _ = mime.ParseMediaType(m)
if s == m {
return true
}
}
return false
}
// SetLimit sets the maximum number of bytes read from input when detecting the MIME type.
// Increasing the limit provides better detection for file formats which store
// their magical numbers towards the end of the file: docx, pptx, xlsx, etc.
// A limit of 0 means the whole input file will be used.
func SetLimit(limit uint32) {
// Using atomic because readLimit can be read at the same time in other goroutine.
atomic.StoreUint32(&readLimit, limit)
}
// Extend adds detection for other file formats.
// It is equivalent to calling Extend() on the root mime type "application/octet-stream".
func Extend(detector func(raw []byte, limit uint32) bool, mime, extension string, aliases ...string) {
root.Extend(detector, mime, extension, aliases...)
}
// Lookup finds a MIME object by its string representation.
// The representation can be the main mime type, or any of its aliases.
func Lookup(mime string) *MIME {
mu.RLock()
defer mu.RUnlock()
return root.lookup(mime)
}
+178
View File
@@ -0,0 +1,178 @@
## 173 Supported MIME types
This file is automatically generated when running tests. Do not edit manually.
Extension | MIME type | Aliases
--------- | --------- | -------
**n/a** | application/octet-stream | -
**.xpm** | image/x-xpixmap | -
**.7z** | application/x-7z-compressed | -
**.zip** | application/zip | application/x-zip, application/x-zip-compressed
**.xlsx** | application/vnd.openxmlformats-officedocument.spreadsheetml.sheet | -
**.docx** | application/vnd.openxmlformats-officedocument.wordprocessingml.document | -
**.pptx** | application/vnd.openxmlformats-officedocument.presentationml.presentation | -
**.epub** | application/epub+zip | -
**.jar** | application/jar | -
**.odt** | application/vnd.oasis.opendocument.text | application/x-vnd.oasis.opendocument.text
**.ott** | application/vnd.oasis.opendocument.text-template | application/x-vnd.oasis.opendocument.text-template
**.ods** | application/vnd.oasis.opendocument.spreadsheet | application/x-vnd.oasis.opendocument.spreadsheet
**.ots** | application/vnd.oasis.opendocument.spreadsheet-template | application/x-vnd.oasis.opendocument.spreadsheet-template
**.odp** | application/vnd.oasis.opendocument.presentation | application/x-vnd.oasis.opendocument.presentation
**.otp** | application/vnd.oasis.opendocument.presentation-template | application/x-vnd.oasis.opendocument.presentation-template
**.odg** | application/vnd.oasis.opendocument.graphics | application/x-vnd.oasis.opendocument.graphics
**.otg** | application/vnd.oasis.opendocument.graphics-template | application/x-vnd.oasis.opendocument.graphics-template
**.odf** | application/vnd.oasis.opendocument.formula | application/x-vnd.oasis.opendocument.formula
**.odc** | application/vnd.oasis.opendocument.chart | application/x-vnd.oasis.opendocument.chart
**.sxc** | application/vnd.sun.xml.calc | -
**.pdf** | application/pdf | application/x-pdf
**.fdf** | application/vnd.fdf | -
**n/a** | application/x-ole-storage | -
**.msi** | application/x-ms-installer | application/x-windows-installer, application/x-msi
**.aaf** | application/octet-stream | -
**.msg** | application/vnd.ms-outlook | -
**.xls** | application/vnd.ms-excel | application/msexcel
**.pub** | application/vnd.ms-publisher | -
**.ppt** | application/vnd.ms-powerpoint | application/mspowerpoint
**.doc** | application/msword | application/vnd.ms-word
**.ps** | application/postscript | -
**.psd** | image/vnd.adobe.photoshop | image/x-psd, application/photoshop
**.p7s** | application/pkcs7-signature | -
**.ogg** | application/ogg | application/x-ogg
**.oga** | audio/ogg | -
**.ogv** | video/ogg | -
**.png** | image/png | -
**.png** | image/vnd.mozilla.apng | -
**.jpg** | image/jpeg | -
**.jxl** | image/jxl | -
**.jp2** | image/jp2 | -
**.jpf** | image/jpx | -
**.jpm** | image/jpm | video/jpm
**.jxs** | image/jxs | -
**.gif** | image/gif | -
**.webp** | image/webp | -
**.exe** | application/vnd.microsoft.portable-executable | -
**n/a** | application/x-elf | -
**n/a** | application/x-object | -
**n/a** | application/x-executable | -
**.so** | application/x-sharedlib | -
**n/a** | application/x-coredump | -
**.a** | application/x-archive | application/x-unix-archive
**.deb** | application/vnd.debian.binary-package | -
**.tar** | application/x-tar | -
**.xar** | application/x-xar | -
**.bz2** | application/x-bzip2 | -
**.fits** | application/fits | -
**.tiff** | image/tiff | -
**.bmp** | image/bmp | image/x-bmp, image/x-ms-bmp
**.ico** | image/x-icon | -
**.mp3** | audio/mpeg | audio/x-mpeg, audio/mp3
**.flac** | audio/flac | -
**.midi** | audio/midi | audio/mid, audio/sp-midi, audio/x-mid, audio/x-midi
**.ape** | audio/ape | -
**.mpc** | audio/musepack | -
**.amr** | audio/amr | audio/amr-nb
**.wav** | audio/wav | audio/x-wav, audio/vnd.wave, audio/wave
**.aiff** | audio/aiff | audio/x-aiff
**.au** | audio/basic | -
**.mpeg** | video/mpeg | -
**.mov** | video/quicktime | -
**.mqv** | video/quicktime | -
**.mp4** | video/mp4 | -
**.webm** | video/webm | audio/webm
**.3gp** | video/3gpp | video/3gp, audio/3gpp
**.3g2** | video/3gpp2 | video/3g2, audio/3gpp2
**.avi** | video/x-msvideo | video/avi, video/msvideo
**.flv** | video/x-flv | -
**.mkv** | video/x-matroska | -
**.asf** | video/x-ms-asf | video/asf, video/x-ms-wmv
**.aac** | audio/aac | -
**.voc** | audio/x-unknown | -
**.mp4** | audio/mp4 | audio/x-m4a, audio/x-mp4a
**.m4a** | audio/x-m4a | -
**.m3u** | application/vnd.apple.mpegurl | audio/mpegurl
**.m4v** | video/x-m4v | -
**.rmvb** | application/vnd.rn-realmedia-vbr | -
**.gz** | application/gzip | application/x-gzip, application/x-gunzip, application/gzipped, application/gzip-compressed, application/x-gzip-compressed, gzip/document
**.class** | application/x-java-applet | -
**.swf** | application/x-shockwave-flash | -
**.crx** | application/x-chrome-extension | -
**.ttf** | font/ttf | font/sfnt, application/x-font-ttf, application/font-sfnt
**.woff** | font/woff | -
**.woff2** | font/woff2 | -
**.otf** | font/otf | -
**.ttc** | font/collection | -
**.eot** | application/vnd.ms-fontobject | -
**.wasm** | application/wasm | -
**.shx** | application/vnd.shx | -
**.shp** | application/vnd.shp | -
**.dbf** | application/x-dbf | -
**.dcm** | application/dicom | -
**.rar** | application/x-rar-compressed | application/x-rar
**.djvu** | image/vnd.djvu | -
**.mobi** | application/x-mobipocket-ebook | -
**.lit** | application/x-ms-reader | -
**.bpg** | image/bpg | -
**.sqlite** | application/vnd.sqlite3 | application/x-sqlite3
**.dwg** | image/vnd.dwg | image/x-dwg, application/acad, application/x-acad, application/autocad_dwg, application/dwg, application/x-dwg, application/x-autocad, drawing/dwg
**.nes** | application/vnd.nintendo.snes.rom | -
**.lnk** | application/x-ms-shortcut | -
**.macho** | application/x-mach-binary | -
**.qcp** | audio/qcelp | -
**.icns** | image/x-icns | -
**.heic** | image/heic | -
**.heic** | image/heic-sequence | -
**.heif** | image/heif | -
**.heif** | image/heif-sequence | -
**.hdr** | image/vnd.radiance | -
**.mrc** | application/marc | -
**.mdb** | application/x-msaccess | -
**.accdb** | application/x-msaccess | -
**.zst** | application/zstd | -
**.cab** | application/vnd.ms-cab-compressed | -
**.rpm** | application/x-rpm | -
**.xz** | application/x-xz | -
**.lz** | application/lzip | application/x-lzip
**.torrent** | application/x-bittorrent | -
**.cpio** | application/x-cpio | -
**n/a** | application/tzif | -
**.xcf** | image/x-xcf | -
**.pat** | image/x-gimp-pat | -
**.gbr** | image/x-gimp-gbr | -
**.glb** | model/gltf-binary | -
**.avif** | image/avif | -
**.cab** | application/x-installshield | -
**.jxr** | image/jxr | image/vnd.ms-photo
**.txt** | text/plain | -
**.html** | text/html | -
**.svg** | image/svg+xml | -
**.xml** | text/xml | -
**.rss** | application/rss+xml | text/rss
**.atom** | application/atom+xml | -
**.x3d** | model/x3d+xml | -
**.kml** | application/vnd.google-earth.kml+xml | -
**.xlf** | application/x-xliff+xml | -
**.dae** | model/vnd.collada+xml | -
**.gml** | application/gml+xml | -
**.gpx** | application/gpx+xml | -
**.tcx** | application/vnd.garmin.tcx+xml | -
**.amf** | application/x-amf | -
**.3mf** | application/vnd.ms-package.3dmanufacturing-3dmodel+xml | -
**.xfdf** | application/vnd.adobe.xfdf | -
**.owl** | application/owl+xml | -
**.php** | text/x-php | -
**.js** | application/javascript | application/x-javascript, text/javascript
**.lua** | text/x-lua | -
**.pl** | text/x-perl | -
**.py** | text/x-python | text/x-script.python, application/x-python
**.json** | application/json | -
**.geojson** | application/geo+json | -
**.har** | application/json | -
**.ndjson** | application/x-ndjson | -
**.rtf** | text/rtf | -
**.srt** | application/x-subrip | application/x-srt, text/x-srt
**.tcl** | text/x-tcl | application/x-tcl
**.csv** | text/csv | -
**.tsv** | text/tab-separated-values | -
**.vcf** | text/vcard | -
**.ics** | text/calendar | -
**.warc** | application/warc | -
**.vtt** | text/vtt | -
+260
View File
@@ -0,0 +1,260 @@
package mimetype
import (
"sync"
"github.com/gabriel-vasile/mimetype/internal/magic"
)
// mimetype stores the list of MIME types in a tree structure with
// "application/octet-stream" at the root of the hierarchy. The hierarchy
// approach minimizes the number of checks that need to be done on the input
// and allows for more precise results once the base type of file has been
// identified.
//
// root is a detector which passes for any slice of bytes.
// When a detector passes the check, the children detectors
// are tried in order to find a more accurate MIME type.
var root = newMIME("application/octet-stream", "",
func([]byte, uint32) bool { return true },
xpm, sevenZ, zip, pdf, fdf, ole, ps, psd, p7s, ogg, png, jpg, jxl, jp2, jpx,
jpm, jxs, gif, webp, exe, elf, ar, tar, xar, bz2, fits, tiff, bmp, ico, mp3, flac,
midi, ape, musePack, amr, wav, aiff, au, mpeg, quickTime, mqv, mp4, webM,
threeGP, threeG2, avi, flv, mkv, asf, aac, voc, aMp4, m4a, m3u, m4v, rmvb,
gzip, class, swf, crx, ttf, woff, woff2, otf, ttc, eot, wasm, shx, dbf, dcm, rar,
djvu, mobi, lit, bpg, sqlite3, dwg, nes, lnk, macho, qcp, icns, heic,
heicSeq, heif, heifSeq, hdr, mrc, mdb, accdb, zstd, cab, rpm, xz, lzip,
torrent, cpio, tzif, xcf, pat, gbr, glb, avif, cabIS, jxr,
// Keep text last because it is the slowest check
text,
)
// errMIME is returned from Detect functions when err is not nil.
// Detect could return root for erroneous cases, but it needs to lock mu in order to do so.
// errMIME is same as root but it does not require locking.
var errMIME = newMIME("application/octet-stream", "", func([]byte, uint32) bool { return false })
// mu guards access to the root MIME tree. Access to root must be synchronized with this lock.
var mu = &sync.RWMutex{}
// The list of nodes appended to the root node.
var (
xz = newMIME("application/x-xz", ".xz", magic.Xz)
gzip = newMIME("application/gzip", ".gz", magic.Gzip).alias(
"application/x-gzip", "application/x-gunzip", "application/gzipped",
"application/gzip-compressed", "application/x-gzip-compressed",
"gzip/document")
sevenZ = newMIME("application/x-7z-compressed", ".7z", magic.SevenZ)
zip = newMIME("application/zip", ".zip", magic.Zip, xlsx, docx, pptx, epub, jar, odt, ods, odp, odg, odf, odc, sxc).
alias("application/x-zip", "application/x-zip-compressed")
tar = newMIME("application/x-tar", ".tar", magic.Tar)
xar = newMIME("application/x-xar", ".xar", magic.Xar)
bz2 = newMIME("application/x-bzip2", ".bz2", magic.Bz2)
pdf = newMIME("application/pdf", ".pdf", magic.Pdf).
alias("application/x-pdf")
fdf = newMIME("application/vnd.fdf", ".fdf", magic.Fdf)
xlsx = newMIME("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ".xlsx", magic.Xlsx)
docx = newMIME("application/vnd.openxmlformats-officedocument.wordprocessingml.document", ".docx", magic.Docx)
pptx = newMIME("application/vnd.openxmlformats-officedocument.presentationml.presentation", ".pptx", magic.Pptx)
epub = newMIME("application/epub+zip", ".epub", magic.Epub)
jar = newMIME("application/jar", ".jar", magic.Jar)
ole = newMIME("application/x-ole-storage", "", magic.Ole, msi, aaf, msg, xls, pub, ppt, doc)
msi = newMIME("application/x-ms-installer", ".msi", magic.Msi).
alias("application/x-windows-installer", "application/x-msi")
aaf = newMIME("application/octet-stream", ".aaf", magic.Aaf)
doc = newMIME("application/msword", ".doc", magic.Doc).
alias("application/vnd.ms-word")
ppt = newMIME("application/vnd.ms-powerpoint", ".ppt", magic.Ppt).
alias("application/mspowerpoint")
pub = newMIME("application/vnd.ms-publisher", ".pub", magic.Pub)
xls = newMIME("application/vnd.ms-excel", ".xls", magic.Xls).
alias("application/msexcel")
msg = newMIME("application/vnd.ms-outlook", ".msg", magic.Msg)
ps = newMIME("application/postscript", ".ps", magic.Ps)
fits = newMIME("application/fits", ".fits", magic.Fits)
ogg = newMIME("application/ogg", ".ogg", magic.Ogg, oggAudio, oggVideo).
alias("application/x-ogg")
oggAudio = newMIME("audio/ogg", ".oga", magic.OggAudio)
oggVideo = newMIME("video/ogg", ".ogv", magic.OggVideo)
text = newMIME("text/plain", ".txt", magic.Text, html, svg, xml, php, js, lua, perl, python, json, ndJSON, rtf, srt, tcl, csv, tsv, vCard, iCalendar, warc, vtt)
xml = newMIME("text/xml", ".xml", magic.XML, rss, atom, x3d, kml, xliff, collada, gml, gpx, tcx, amf, threemf, xfdf, owl2)
json = newMIME("application/json", ".json", magic.JSON, geoJSON, har)
har = newMIME("application/json", ".har", magic.HAR)
csv = newMIME("text/csv", ".csv", magic.Csv)
tsv = newMIME("text/tab-separated-values", ".tsv", magic.Tsv)
geoJSON = newMIME("application/geo+json", ".geojson", magic.GeoJSON)
ndJSON = newMIME("application/x-ndjson", ".ndjson", magic.NdJSON)
html = newMIME("text/html", ".html", magic.HTML)
php = newMIME("text/x-php", ".php", magic.Php)
rtf = newMIME("text/rtf", ".rtf", magic.Rtf)
js = newMIME("application/javascript", ".js", magic.Js).
alias("application/x-javascript", "text/javascript")
srt = newMIME("application/x-subrip", ".srt", magic.Srt).
alias("application/x-srt", "text/x-srt")
vtt = newMIME("text/vtt", ".vtt", magic.Vtt)
lua = newMIME("text/x-lua", ".lua", magic.Lua)
perl = newMIME("text/x-perl", ".pl", magic.Perl)
python = newMIME("text/x-python", ".py", magic.Python).
alias("text/x-script.python", "application/x-python")
tcl = newMIME("text/x-tcl", ".tcl", magic.Tcl).
alias("application/x-tcl")
vCard = newMIME("text/vcard", ".vcf", magic.VCard)
iCalendar = newMIME("text/calendar", ".ics", magic.ICalendar)
svg = newMIME("image/svg+xml", ".svg", magic.Svg)
rss = newMIME("application/rss+xml", ".rss", magic.Rss).
alias("text/rss")
owl2 = newMIME("application/owl+xml", ".owl", magic.Owl2)
atom = newMIME("application/atom+xml", ".atom", magic.Atom)
x3d = newMIME("model/x3d+xml", ".x3d", magic.X3d)
kml = newMIME("application/vnd.google-earth.kml+xml", ".kml", magic.Kml)
xliff = newMIME("application/x-xliff+xml", ".xlf", magic.Xliff)
collada = newMIME("model/vnd.collada+xml", ".dae", magic.Collada)
gml = newMIME("application/gml+xml", ".gml", magic.Gml)
gpx = newMIME("application/gpx+xml", ".gpx", magic.Gpx)
tcx = newMIME("application/vnd.garmin.tcx+xml", ".tcx", magic.Tcx)
amf = newMIME("application/x-amf", ".amf", magic.Amf)
threemf = newMIME("application/vnd.ms-package.3dmanufacturing-3dmodel+xml", ".3mf", magic.Threemf)
png = newMIME("image/png", ".png", magic.Png, apng)
apng = newMIME("image/vnd.mozilla.apng", ".png", magic.Apng)
jpg = newMIME("image/jpeg", ".jpg", magic.Jpg)
jxl = newMIME("image/jxl", ".jxl", magic.Jxl)
jp2 = newMIME("image/jp2", ".jp2", magic.Jp2)
jpx = newMIME("image/jpx", ".jpf", magic.Jpx)
jpm = newMIME("image/jpm", ".jpm", magic.Jpm).
alias("video/jpm")
jxs = newMIME("image/jxs", ".jxs", magic.Jxs)
xpm = newMIME("image/x-xpixmap", ".xpm", magic.Xpm)
bpg = newMIME("image/bpg", ".bpg", magic.Bpg)
gif = newMIME("image/gif", ".gif", magic.Gif)
webp = newMIME("image/webp", ".webp", magic.Webp)
tiff = newMIME("image/tiff", ".tiff", magic.Tiff)
bmp = newMIME("image/bmp", ".bmp", magic.Bmp).
alias("image/x-bmp", "image/x-ms-bmp")
ico = newMIME("image/x-icon", ".ico", magic.Ico)
icns = newMIME("image/x-icns", ".icns", magic.Icns)
psd = newMIME("image/vnd.adobe.photoshop", ".psd", magic.Psd).
alias("image/x-psd", "application/photoshop")
heic = newMIME("image/heic", ".heic", magic.Heic)
heicSeq = newMIME("image/heic-sequence", ".heic", magic.HeicSequence)
heif = newMIME("image/heif", ".heif", magic.Heif)
heifSeq = newMIME("image/heif-sequence", ".heif", magic.HeifSequence)
hdr = newMIME("image/vnd.radiance", ".hdr", magic.Hdr)
avif = newMIME("image/avif", ".avif", magic.AVIF)
mp3 = newMIME("audio/mpeg", ".mp3", magic.Mp3).
alias("audio/x-mpeg", "audio/mp3")
flac = newMIME("audio/flac", ".flac", magic.Flac)
midi = newMIME("audio/midi", ".midi", magic.Midi).
alias("audio/mid", "audio/sp-midi", "audio/x-mid", "audio/x-midi")
ape = newMIME("audio/ape", ".ape", magic.Ape)
musePack = newMIME("audio/musepack", ".mpc", magic.MusePack)
wav = newMIME("audio/wav", ".wav", magic.Wav).
alias("audio/x-wav", "audio/vnd.wave", "audio/wave")
aiff = newMIME("audio/aiff", ".aiff", magic.Aiff).alias("audio/x-aiff")
au = newMIME("audio/basic", ".au", magic.Au)
amr = newMIME("audio/amr", ".amr", magic.Amr).
alias("audio/amr-nb")
aac = newMIME("audio/aac", ".aac", magic.AAC)
voc = newMIME("audio/x-unknown", ".voc", magic.Voc)
aMp4 = newMIME("audio/mp4", ".mp4", magic.AMp4).
alias("audio/x-m4a", "audio/x-mp4a")
m4a = newMIME("audio/x-m4a", ".m4a", magic.M4a)
m3u = newMIME("application/vnd.apple.mpegurl", ".m3u", magic.M3u).
alias("audio/mpegurl")
m4v = newMIME("video/x-m4v", ".m4v", magic.M4v)
mp4 = newMIME("video/mp4", ".mp4", magic.Mp4)
webM = newMIME("video/webm", ".webm", magic.WebM).
alias("audio/webm")
mpeg = newMIME("video/mpeg", ".mpeg", magic.Mpeg)
quickTime = newMIME("video/quicktime", ".mov", magic.QuickTime)
mqv = newMIME("video/quicktime", ".mqv", magic.Mqv)
threeGP = newMIME("video/3gpp", ".3gp", magic.ThreeGP).
alias("video/3gp", "audio/3gpp")
threeG2 = newMIME("video/3gpp2", ".3g2", magic.ThreeG2).
alias("video/3g2", "audio/3gpp2")
avi = newMIME("video/x-msvideo", ".avi", magic.Avi).
alias("video/avi", "video/msvideo")
flv = newMIME("video/x-flv", ".flv", magic.Flv)
mkv = newMIME("video/x-matroska", ".mkv", magic.Mkv)
asf = newMIME("video/x-ms-asf", ".asf", magic.Asf).
alias("video/asf", "video/x-ms-wmv")
rmvb = newMIME("application/vnd.rn-realmedia-vbr", ".rmvb", magic.Rmvb)
class = newMIME("application/x-java-applet", ".class", magic.Class)
swf = newMIME("application/x-shockwave-flash", ".swf", magic.SWF)
crx = newMIME("application/x-chrome-extension", ".crx", magic.CRX)
ttf = newMIME("font/ttf", ".ttf", magic.Ttf).
alias("font/sfnt", "application/x-font-ttf", "application/font-sfnt")
woff = newMIME("font/woff", ".woff", magic.Woff)
woff2 = newMIME("font/woff2", ".woff2", magic.Woff2)
otf = newMIME("font/otf", ".otf", magic.Otf)
ttc = newMIME("font/collection", ".ttc", magic.Ttc)
eot = newMIME("application/vnd.ms-fontobject", ".eot", magic.Eot)
wasm = newMIME("application/wasm", ".wasm", magic.Wasm)
shp = newMIME("application/vnd.shp", ".shp", magic.Shp)
shx = newMIME("application/vnd.shx", ".shx", magic.Shx, shp)
dbf = newMIME("application/x-dbf", ".dbf", magic.Dbf)
exe = newMIME("application/vnd.microsoft.portable-executable", ".exe", magic.Exe)
elf = newMIME("application/x-elf", "", magic.Elf, elfObj, elfExe, elfLib, elfDump)
elfObj = newMIME("application/x-object", "", magic.ElfObj)
elfExe = newMIME("application/x-executable", "", magic.ElfExe)
elfLib = newMIME("application/x-sharedlib", ".so", magic.ElfLib)
elfDump = newMIME("application/x-coredump", "", magic.ElfDump)
ar = newMIME("application/x-archive", ".a", magic.Ar, deb).
alias("application/x-unix-archive")
deb = newMIME("application/vnd.debian.binary-package", ".deb", magic.Deb)
rpm = newMIME("application/x-rpm", ".rpm", magic.RPM)
dcm = newMIME("application/dicom", ".dcm", magic.Dcm)
odt = newMIME("application/vnd.oasis.opendocument.text", ".odt", magic.Odt, ott).
alias("application/x-vnd.oasis.opendocument.text")
ott = newMIME("application/vnd.oasis.opendocument.text-template", ".ott", magic.Ott).
alias("application/x-vnd.oasis.opendocument.text-template")
ods = newMIME("application/vnd.oasis.opendocument.spreadsheet", ".ods", magic.Ods, ots).
alias("application/x-vnd.oasis.opendocument.spreadsheet")
ots = newMIME("application/vnd.oasis.opendocument.spreadsheet-template", ".ots", magic.Ots).
alias("application/x-vnd.oasis.opendocument.spreadsheet-template")
odp = newMIME("application/vnd.oasis.opendocument.presentation", ".odp", magic.Odp, otp).
alias("application/x-vnd.oasis.opendocument.presentation")
otp = newMIME("application/vnd.oasis.opendocument.presentation-template", ".otp", magic.Otp).
alias("application/x-vnd.oasis.opendocument.presentation-template")
odg = newMIME("application/vnd.oasis.opendocument.graphics", ".odg", magic.Odg, otg).
alias("application/x-vnd.oasis.opendocument.graphics")
otg = newMIME("application/vnd.oasis.opendocument.graphics-template", ".otg", magic.Otg).
alias("application/x-vnd.oasis.opendocument.graphics-template")
odf = newMIME("application/vnd.oasis.opendocument.formula", ".odf", magic.Odf).
alias("application/x-vnd.oasis.opendocument.formula")
odc = newMIME("application/vnd.oasis.opendocument.chart", ".odc", magic.Odc).
alias("application/x-vnd.oasis.opendocument.chart")
sxc = newMIME("application/vnd.sun.xml.calc", ".sxc", magic.Sxc)
rar = newMIME("application/x-rar-compressed", ".rar", magic.RAR).
alias("application/x-rar")
djvu = newMIME("image/vnd.djvu", ".djvu", magic.DjVu)
mobi = newMIME("application/x-mobipocket-ebook", ".mobi", magic.Mobi)
lit = newMIME("application/x-ms-reader", ".lit", magic.Lit)
sqlite3 = newMIME("application/vnd.sqlite3", ".sqlite", magic.Sqlite).
alias("application/x-sqlite3")
dwg = newMIME("image/vnd.dwg", ".dwg", magic.Dwg).
alias("image/x-dwg", "application/acad", "application/x-acad",
"application/autocad_dwg", "application/dwg", "application/x-dwg",
"application/x-autocad", "drawing/dwg")
warc = newMIME("application/warc", ".warc", magic.Warc)
nes = newMIME("application/vnd.nintendo.snes.rom", ".nes", magic.Nes)
lnk = newMIME("application/x-ms-shortcut", ".lnk", magic.Lnk)
macho = newMIME("application/x-mach-binary", ".macho", magic.MachO)
qcp = newMIME("audio/qcelp", ".qcp", magic.Qcp)
mrc = newMIME("application/marc", ".mrc", magic.Marc)
mdb = newMIME("application/x-msaccess", ".mdb", magic.MsAccessMdb)
accdb = newMIME("application/x-msaccess", ".accdb", magic.MsAccessAce)
zstd = newMIME("application/zstd", ".zst", magic.Zstd)
cab = newMIME("application/vnd.ms-cab-compressed", ".cab", magic.Cab)
cabIS = newMIME("application/x-installshield", ".cab", magic.InstallShieldCab)
lzip = newMIME("application/lzip", ".lz", magic.Lzip).alias("application/x-lzip")
torrent = newMIME("application/x-bittorrent", ".torrent", magic.Torrent)
cpio = newMIME("application/x-cpio", ".cpio", magic.Cpio)
tzif = newMIME("application/tzif", "", magic.TzIf)
p7s = newMIME("application/pkcs7-signature", ".p7s", magic.P7s)
xcf = newMIME("image/x-xcf", ".xcf", magic.Xcf)
pat = newMIME("image/x-gimp-pat", ".pat", magic.Pat)
gbr = newMIME("image/x-gimp-gbr", ".gbr", magic.Gbr)
xfdf = newMIME("application/vnd.adobe.xfdf", ".xfdf", magic.Xfdf)
glb = newMIME("model/gltf-binary", ".glb", magic.Glb)
jxr = newMIME("image/jxr", ".jxr", magic.Jxr).alias("image/vnd.ms-photo")
)
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build !windows
// +build !windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build !windows
// +build !windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package ole
+5 -5
View File
@@ -108,9 +108,9 @@ type GUID struct {
//
// The supplied string may be in any of these formats:
//
// XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
// XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
// {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}
// XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
// XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
// {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}
//
// The conversion of the supplied string is not case-sensitive.
func NewGUID(guid string) *GUID {
@@ -216,11 +216,11 @@ func decodeHexChar(c byte) (byte, bool) {
// String converts the GUID to string form. It will adhere to this pattern:
//
// {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}
// {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}
//
// If the GUID is nil, the string representation of an empty GUID is returned:
//
// {00000000-0000-0000-0000-000000000000}
// {00000000-0000-0000-0000-000000000000}
func (guid *GUID) String() string {
if guid == nil {
return emptyGUID
-1
View File
@@ -1,4 +1,3 @@
//go:build !windows
// +build !windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build !windows
// +build !windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build !windows
// +build !windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build !windows
// +build !windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build !windows
// +build !windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build !windows
// +build !windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build !windows
// +build !windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build !windows
// +build !windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package oleutil
-1
View File
@@ -1,4 +1,3 @@
//go:build !windows
// +build !windows
package oleutil
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package oleutil
-1
View File
@@ -1,7 +1,6 @@
// This file is here so go get succeeds as without it errors with:
// no buildable Go source files in ...
//
//go:build !windows
// +build !windows
package oleutil
-1
View File
@@ -1,4 +1,3 @@
//go:build !windows
// +build !windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package ole
+1 -1
View File
@@ -84,7 +84,7 @@ func (sac *SafeArrayConversion) ToValueArray() (values []interface{}) {
safeArrayGetElement(sac.Array, i, unsafe.Pointer(&v))
values[i] = v
case VT_BSTR:
v, _ := safeArrayGetElementString(sac.Array, i)
v , _ := safeArrayGetElementString(sac.Array, i)
values[i] = v
case VT_VARIANT:
var v VARIANT
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build 386
// +build 386
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build amd64
// +build amd64
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build arm
// +build arm
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows && 386
// +build windows,386
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows && amd64
// +build windows,amd64
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows && arm
// +build windows,arm
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build ppc64le
// +build ppc64le
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build s390x
// +build s390x
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build windows
// +build windows
package ole
-1
View File
@@ -1,4 +1,3 @@
//go:build !windows
// +build !windows
package ole
+24
View File
@@ -0,0 +1,24 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
+26
View File
@@ -0,0 +1,26 @@
language: go
go:
- 1.13.1
- tip
matrix:
allow_failures:
- go: tip
notifications:
email:
recipients: dean.karn@gmail.com
on_success: change
on_failure: always
before_install:
- go install github.com/mattn/goveralls
# Only clone the most recent commit.
git:
depth: 1
script:
- go test -v -race -covermode=atomic -coverprofile=coverage.coverprofile ./...
after_success: |
goveralls -coverprofile=coverage.coverprofile -service travis-ci -repotoken $COVERALLS_TOKEN
+21
View File
@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2016 Go Playground
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+170
View File
@@ -0,0 +1,170 @@
## locales
<img align="right" src="https://raw.githubusercontent.com/go-playground/locales/master/logo.png">![Project status](https://img.shields.io/badge/version-0.14.1-green.svg)
[![Build Status](https://travis-ci.org/go-playground/locales.svg?branch=master)](https://travis-ci.org/go-playground/locales)
[![GoDoc](https://godoc.org/github.com/go-playground/locales?status.svg)](https://godoc.org/github.com/go-playground/locales)
![License](https://img.shields.io/dub/l/vibe-d.svg)
Locales is a set of locales generated from the [Unicode CLDR Project](http://cldr.unicode.org/) which can be used independently or within
an i18n package; these were built for use with, but not exclusive to, [Universal Translator](https://github.com/go-playground/universal-translator).
Features
--------
- [x] Rules generated from the latest [CLDR](http://cldr.unicode.org/index/downloads) data, v36.0.1
- [x] Contains Cardinal, Ordinal and Range Plural Rules
- [x] Contains Month, Weekday and Timezone translations built in
- [x] Contains Date & Time formatting functions
- [x] Contains Number, Currency, Accounting and Percent formatting functions
- [x] Supports the "Gregorian" calendar only ( my time isn't unlimited, had to draw the line somewhere )
Full Tests
--------------------
I could sure use your help adding tests for every locale, it is a huge undertaking and I just don't have the free time to do it all at the moment;
any help would be **greatly appreciated!!!!** please see [issue](https://github.com/go-playground/locales/issues/1) for details.
Installation
-----------
Use go get
```shell
go get github.com/go-playground/locales
```
NOTES
--------
You'll notice most return types are []byte, this is because most of the time the results will be concatenated with a larger body
of text and can avoid some allocations if already appending to a byte array, otherwise just cast as string.
Usage
-------
```go
package main
import (
"fmt"
"time"
"github.com/go-playground/locales/currency"
"github.com/go-playground/locales/en_CA"
)
func main() {
loc, _ := time.LoadLocation("America/Toronto")
datetime := time.Date(2016, 02, 03, 9, 0, 1, 0, loc)
l := en_CA.New()
// Dates
fmt.Println(l.FmtDateFull(datetime))
fmt.Println(l.FmtDateLong(datetime))
fmt.Println(l.FmtDateMedium(datetime))
fmt.Println(l.FmtDateShort(datetime))
// Times
fmt.Println(l.FmtTimeFull(datetime))
fmt.Println(l.FmtTimeLong(datetime))
fmt.Println(l.FmtTimeMedium(datetime))
fmt.Println(l.FmtTimeShort(datetime))
// Months Wide
fmt.Println(l.MonthWide(time.January))
fmt.Println(l.MonthWide(time.February))
fmt.Println(l.MonthWide(time.March))
// ...
// Months Abbreviated
fmt.Println(l.MonthAbbreviated(time.January))
fmt.Println(l.MonthAbbreviated(time.February))
fmt.Println(l.MonthAbbreviated(time.March))
// ...
// Months Narrow
fmt.Println(l.MonthNarrow(time.January))
fmt.Println(l.MonthNarrow(time.February))
fmt.Println(l.MonthNarrow(time.March))
// ...
// Weekdays Wide
fmt.Println(l.WeekdayWide(time.Sunday))
fmt.Println(l.WeekdayWide(time.Monday))
fmt.Println(l.WeekdayWide(time.Tuesday))
// ...
// Weekdays Abbreviated
fmt.Println(l.WeekdayAbbreviated(time.Sunday))
fmt.Println(l.WeekdayAbbreviated(time.Monday))
fmt.Println(l.WeekdayAbbreviated(time.Tuesday))
// ...
// Weekdays Short
fmt.Println(l.WeekdayShort(time.Sunday))
fmt.Println(l.WeekdayShort(time.Monday))
fmt.Println(l.WeekdayShort(time.Tuesday))
// ...
// Weekdays Narrow
fmt.Println(l.WeekdayNarrow(time.Sunday))
fmt.Println(l.WeekdayNarrow(time.Monday))
fmt.Println(l.WeekdayNarrow(time.Tuesday))
// ...
var f64 float64
f64 = -10356.4523
// Number
fmt.Println(l.FmtNumber(f64, 2))
// Currency
fmt.Println(l.FmtCurrency(f64, 2, currency.CAD))
fmt.Println(l.FmtCurrency(f64, 2, currency.USD))
// Accounting
fmt.Println(l.FmtAccounting(f64, 2, currency.CAD))
fmt.Println(l.FmtAccounting(f64, 2, currency.USD))
f64 = 78.12
// Percent
fmt.Println(l.FmtPercent(f64, 0))
// Plural Rules for locale, so you know what rules you must cover
fmt.Println(l.PluralsCardinal())
fmt.Println(l.PluralsOrdinal())
// Cardinal Plural Rules
fmt.Println(l.CardinalPluralRule(1, 0))
fmt.Println(l.CardinalPluralRule(1.0, 0))
fmt.Println(l.CardinalPluralRule(1.0, 1))
fmt.Println(l.CardinalPluralRule(3, 0))
// Ordinal Plural Rules
fmt.Println(l.OrdinalPluralRule(21, 0)) // 21st
fmt.Println(l.OrdinalPluralRule(22, 0)) // 22nd
fmt.Println(l.OrdinalPluralRule(33, 0)) // 33rd
fmt.Println(l.OrdinalPluralRule(34, 0)) // 34th
// Range Plural Rules
fmt.Println(l.RangePluralRule(1, 0, 1, 0)) // 1-1
fmt.Println(l.RangePluralRule(1, 0, 2, 0)) // 1-2
fmt.Println(l.RangePluralRule(5, 0, 8, 0)) // 5-8
}
```
NOTES:
-------
These rules were generated from the [Unicode CLDR Project](http://cldr.unicode.org/), if you encounter any issues
I strongly encourage contributing to the CLDR project to get the locale information corrected and the next time
these locales are regenerated the fix will come with.
I do however realize that time constraints are often important and so there are two options:
1. Create your own locale, copy, paste and modify, and ensure it complies with the `Translator` interface.
2. Add an exception in the locale generation code directly and once regenerated, fix will be in place.
Please to not make fixes inside the locale files, they WILL get overwritten when the locales are regenerated.
License
------
Distributed under MIT License, please see license file in code for more details.
+311
View File
@@ -0,0 +1,311 @@
package currency
// Type is the currency type associated with the locales currency enum
type Type int
// locale currencies
const (
ADP Type = iota
AED
AFA
AFN
ALK
ALL
AMD
ANG
AOA
AOK
AON
AOR
ARA
ARL
ARM
ARP
ARS
ATS
AUD
AWG
AZM
AZN
BAD
BAM
BAN
BBD
BDT
BEC
BEF
BEL
BGL
BGM
BGN
BGO
BHD
BIF
BMD
BND
BOB
BOL
BOP
BOV
BRB
BRC
BRE
BRL
BRN
BRR
BRZ
BSD
BTN
BUK
BWP
BYB
BYN
BYR
BZD
CAD
CDF
CHE
CHF
CHW
CLE
CLF
CLP
CNH
CNX
CNY
COP
COU
CRC
CSD
CSK
CUC
CUP
CVE
CYP
CZK
DDM
DEM
DJF
DKK
DOP
DZD
ECS
ECV
EEK
EGP
ERN
ESA
ESB
ESP
ETB
EUR
FIM
FJD
FKP
FRF
GBP
GEK
GEL
GHC
GHS
GIP
GMD
GNF
GNS
GQE
GRD
GTQ
GWE
GWP
GYD
HKD
HNL
HRD
HRK
HTG
HUF
IDR
IEP
ILP
ILR
ILS
INR
IQD
IRR
ISJ
ISK
ITL
JMD
JOD
JPY
KES
KGS
KHR
KMF
KPW
KRH
KRO
KRW
KWD
KYD
KZT
LAK
LBP
LKR
LRD
LSL
LTL
LTT
LUC
LUF
LUL
LVL
LVR
LYD
MAD
MAF
MCF
MDC
MDL
MGA
MGF
MKD
MKN
MLF
MMK
MNT
MOP
MRO
MRU
MTL
MTP
MUR
MVP
MVR
MWK
MXN
MXP
MXV
MYR
MZE
MZM
MZN
NAD
NGN
NIC
NIO
NLG
NOK
NPR
NZD
OMR
PAB
PEI
PEN
PES
PGK
PHP
PKR
PLN
PLZ
PTE
PYG
QAR
RHD
ROL
RON
RSD
RUB
RUR
RWF
SAR
SBD
SCR
SDD
SDG
SDP
SEK
SGD
SHP
SIT
SKK
SLL
SOS
SRD
SRG
SSP
STD
STN
SUR
SVC
SYP
SZL
THB
TJR
TJS
TMM
TMT
TND
TOP
TPE
TRL
TRY
TTD
TWD
TZS
UAH
UAK
UGS
UGX
USD
USN
USS
UYI
UYP
UYU
UYW
UZS
VEB
VEF
VES
VND
VNN
VUV
WST
XAF
XAG
XAU
XBA
XBB
XBC
XBD
XCD
XDR
XEU
XFO
XFU
XOF
XPD
XPF
XPT
XRE
XSU
XTS
XUA
XXX
YDD
YER
YUD
YUM
YUN
YUR
ZAL
ZAR
ZMK
ZMW
ZRN
ZRZ
ZWD
ZWL
ZWR
)
Binary file not shown.
+293
View File
@@ -0,0 +1,293 @@
package locales
import (
"strconv"
"time"
"github.com/go-playground/locales/currency"
)
// // ErrBadNumberValue is returned when the number passed for
// // plural rule determination cannot be parsed
// type ErrBadNumberValue struct {
// NumberValue string
// InnerError error
// }
// // Error returns ErrBadNumberValue error string
// func (e *ErrBadNumberValue) Error() string {
// return fmt.Sprintf("Invalid Number Value '%s' %s", e.NumberValue, e.InnerError)
// }
// var _ error = new(ErrBadNumberValue)
// PluralRule denotes the type of plural rules
type PluralRule int
// PluralRule's
const (
PluralRuleUnknown PluralRule = iota
PluralRuleZero // zero
PluralRuleOne // one - singular
PluralRuleTwo // two - dual
PluralRuleFew // few - paucal
PluralRuleMany // many - also used for fractions if they have a separate class
PluralRuleOther // other - required—general plural form—also used if the language only has a single form
)
const (
pluralsString = "UnknownZeroOneTwoFewManyOther"
)
// Translator encapsulates an instance of a locale
// NOTE: some values are returned as a []byte just in case the caller
// wishes to add more and can help avoid allocations; otherwise just cast as string
type Translator interface {
// The following Functions are for overriding, debugging or developing
// with a Translator Locale
// Locale returns the string value of the translator
Locale() string
// returns an array of cardinal plural rules associated
// with this translator
PluralsCardinal() []PluralRule
// returns an array of ordinal plural rules associated
// with this translator
PluralsOrdinal() []PluralRule
// returns an array of range plural rules associated
// with this translator
PluralsRange() []PluralRule
// returns the cardinal PluralRule given 'num' and digits/precision of 'v' for locale
CardinalPluralRule(num float64, v uint64) PluralRule
// returns the ordinal PluralRule given 'num' and digits/precision of 'v' for locale
OrdinalPluralRule(num float64, v uint64) PluralRule
// returns the ordinal PluralRule given 'num1', 'num2' and digits/precision of 'v1' and 'v2' for locale
RangePluralRule(num1 float64, v1 uint64, num2 float64, v2 uint64) PluralRule
// returns the locales abbreviated month given the 'month' provided
MonthAbbreviated(month time.Month) string
// returns the locales abbreviated months
MonthsAbbreviated() []string
// returns the locales narrow month given the 'month' provided
MonthNarrow(month time.Month) string
// returns the locales narrow months
MonthsNarrow() []string
// returns the locales wide month given the 'month' provided
MonthWide(month time.Month) string
// returns the locales wide months
MonthsWide() []string
// returns the locales abbreviated weekday given the 'weekday' provided
WeekdayAbbreviated(weekday time.Weekday) string
// returns the locales abbreviated weekdays
WeekdaysAbbreviated() []string
// returns the locales narrow weekday given the 'weekday' provided
WeekdayNarrow(weekday time.Weekday) string
// WeekdaysNarrowreturns the locales narrow weekdays
WeekdaysNarrow() []string
// returns the locales short weekday given the 'weekday' provided
WeekdayShort(weekday time.Weekday) string
// returns the locales short weekdays
WeekdaysShort() []string
// returns the locales wide weekday given the 'weekday' provided
WeekdayWide(weekday time.Weekday) string
// returns the locales wide weekdays
WeekdaysWide() []string
// The following Functions are common Formatting functionsfor the Translator's Locale
// returns 'num' with digits/precision of 'v' for locale and handles both Whole and Real numbers based on 'v'
FmtNumber(num float64, v uint64) string
// returns 'num' with digits/precision of 'v' for locale and handles both Whole and Real numbers based on 'v'
// NOTE: 'num' passed into FmtPercent is assumed to be in percent already
FmtPercent(num float64, v uint64) string
// returns the currency representation of 'num' with digits/precision of 'v' for locale
FmtCurrency(num float64, v uint64, currency currency.Type) string
// returns the currency representation of 'num' with digits/precision of 'v' for locale
// in accounting notation.
FmtAccounting(num float64, v uint64, currency currency.Type) string
// returns the short date representation of 't' for locale
FmtDateShort(t time.Time) string
// returns the medium date representation of 't' for locale
FmtDateMedium(t time.Time) string
// returns the long date representation of 't' for locale
FmtDateLong(t time.Time) string
// returns the full date representation of 't' for locale
FmtDateFull(t time.Time) string
// returns the short time representation of 't' for locale
FmtTimeShort(t time.Time) string
// returns the medium time representation of 't' for locale
FmtTimeMedium(t time.Time) string
// returns the long time representation of 't' for locale
FmtTimeLong(t time.Time) string
// returns the full time representation of 't' for locale
FmtTimeFull(t time.Time) string
}
// String returns the string value of PluralRule
func (p PluralRule) String() string {
switch p {
case PluralRuleZero:
return pluralsString[7:11]
case PluralRuleOne:
return pluralsString[11:14]
case PluralRuleTwo:
return pluralsString[14:17]
case PluralRuleFew:
return pluralsString[17:20]
case PluralRuleMany:
return pluralsString[20:24]
case PluralRuleOther:
return pluralsString[24:]
default:
return pluralsString[:7]
}
}
//
// Precision Notes:
//
// must specify a precision >= 0, and here is why https://play.golang.org/p/LyL90U0Vyh
//
// v := float64(3.141)
// i := float64(int64(v))
//
// fmt.Println(v - i)
//
// or
//
// s := strconv.FormatFloat(v-i, 'f', -1, 64)
// fmt.Println(s)
//
// these will not print what you'd expect: 0.14100000000000001
// and so this library requires a precision to be specified, or
// inaccurate plural rules could be applied.
//
//
//
// n - absolute value of the source number (integer and decimals).
// i - integer digits of n.
// v - number of visible fraction digits in n, with trailing zeros.
// w - number of visible fraction digits in n, without trailing zeros.
// f - visible fractional digits in n, with trailing zeros.
// t - visible fractional digits in n, without trailing zeros.
//
//
// Func(num float64, v uint64) // v = digits/precision and prevents -1 as a special case as this can lead to very unexpected behaviour, see precision note's above.
//
// n := math.Abs(num)
// i := int64(n)
// v := v
//
//
// w := strconv.FormatFloat(num-float64(i), 'f', int(v), 64) // then parse backwards on string until no more zero's....
// f := strconv.FormatFloat(n, 'f', int(v), 64) // then turn everything after decimal into an int64
// t := strconv.FormatFloat(n, 'f', int(v), 64) // then parse backwards on string until no more zero's....
//
//
//
// General Inclusion Rules
// - v will always be available inherently
// - all require n
// - w requires i
//
// W returns the number of visible fraction digits in N, without trailing zeros.
func W(n float64, v uint64) (w int64) {
s := strconv.FormatFloat(n-float64(int64(n)), 'f', int(v), 64)
// with either be '0' or '0.xxxx', so if 1 then w will be zero
// otherwise need to parse
if len(s) != 1 {
s = s[2:]
end := len(s) + 1
for i := end; i >= 0; i-- {
if s[i] != '0' {
end = i + 1
break
}
}
w = int64(len(s[:end]))
}
return
}
// F returns the visible fractional digits in N, with trailing zeros.
func F(n float64, v uint64) (f int64) {
s := strconv.FormatFloat(n-float64(int64(n)), 'f', int(v), 64)
// with either be '0' or '0.xxxx', so if 1 then f will be zero
// otherwise need to parse
if len(s) != 1 {
// ignoring error, because it can't fail as we generated
// the string internally from a real number
f, _ = strconv.ParseInt(s[2:], 10, 64)
}
return
}
// T returns the visible fractional digits in N, without trailing zeros.
func T(n float64, v uint64) (t int64) {
s := strconv.FormatFloat(n-float64(int64(n)), 'f', int(v), 64)
// with either be '0' or '0.xxxx', so if 1 then t will be zero
// otherwise need to parse
if len(s) != 1 {
s = s[2:]
end := len(s) + 1
for i := end; i >= 0; i-- {
if s[i] != '0' {
end = i + 1
break
}
}
// ignoring error, because it can't fail as we generated
// the string internally from a real number
t, _ = strconv.ParseInt(s[:end], 10, 64)
}
return
}
@@ -0,0 +1,25 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
*.coverprofile
+27
View File
@@ -0,0 +1,27 @@
language: go
go:
- 1.13.4
- tip
matrix:
allow_failures:
- go: tip
notifications:
email:
recipients: dean.karn@gmail.com
on_success: change
on_failure: always
before_install:
- go install github.com/mattn/goveralls
# Only clone the most recent commit.
git:
depth: 1
script:
- go test -v -race -covermode=atomic -coverprofile=coverage.coverprofile ./...
after_success: |
[ $TRAVIS_GO_VERSION = 1.13.4 ] &&
goveralls -coverprofile=coverage.coverprofile -service travis-ci -repotoken $COVERALLS_TOKEN
+21
View File
@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2016 Go Playground
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+18
View File
@@ -0,0 +1,18 @@
GOCMD=GO111MODULE=on go
linters-install:
@golangci-lint --version >/dev/null 2>&1 || { \
echo "installing linting tools..."; \
curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.41.1; \
}
lint: linters-install
golangci-lint run
test:
$(GOCMD) test -cover -race ./...
bench:
$(GOCMD) test -bench=. -benchmem ./...
.PHONY: test lint linters-install
+87
View File
@@ -0,0 +1,87 @@
## universal-translator
<img align="right" src="https://raw.githubusercontent.com/go-playground/universal-translator/master/logo.png">![Project status](https://img.shields.io/badge/version-0.18.1-green.svg)
[![Coverage Status](https://coveralls.io/repos/github/go-playground/universal-translator/badge.svg)](https://coveralls.io/github/go-playground/universal-translator)
[![Go Report Card](https://goreportcard.com/badge/github.com/go-playground/universal-translator)](https://goreportcard.com/report/github.com/go-playground/universal-translator)
[![GoDoc](https://godoc.org/github.com/go-playground/universal-translator?status.svg)](https://godoc.org/github.com/go-playground/universal-translator)
![License](https://img.shields.io/dub/l/vibe-d.svg)
Universal Translator is an i18n Translator for Go/Golang using CLDR data + pluralization rules
Why another i18n library?
--------------------------
Because none of the plural rules seem to be correct out there, including the previous implementation of this package,
so I took it upon myself to create [locales](https://github.com/go-playground/locales) for everyone to use; this package
is a thin wrapper around [locales](https://github.com/go-playground/locales) in order to store and translate text for
use in your applications.
Features
--------
- [x] Rules generated from the [CLDR](http://cldr.unicode.org/index/downloads) data, v36.0.1
- [x] Contains Cardinal, Ordinal and Range Plural Rules
- [x] Contains Month, Weekday and Timezone translations built in
- [x] Contains Date & Time formatting functions
- [x] Contains Number, Currency, Accounting and Percent formatting functions
- [x] Supports the "Gregorian" calendar only ( my time isn't unlimited, had to draw the line somewhere )
- [x] Support loading translations from files
- [x] Exporting translations to file(s), mainly for getting them professionally translated
- [ ] Code Generation for translation files -> Go code.. i.e. after it has been professionally translated
- [ ] Tests for all languages, I need help with this, please see [here](https://github.com/go-playground/locales/issues/1)
Installation
-----------
Use go get
```shell
go get github.com/go-playground/universal-translator
```
Usage & Documentation
-------
Please see https://godoc.org/github.com/go-playground/universal-translator for usage docs
##### Examples:
- [Basic](https://github.com/go-playground/universal-translator/tree/master/_examples/basic)
- [Full - no files](https://github.com/go-playground/universal-translator/tree/master/_examples/full-no-files)
- [Full - with files](https://github.com/go-playground/universal-translator/tree/master/_examples/full-with-files)
File formatting
--------------
All types, Plain substitution, Cardinal, Ordinal and Range translations can all be contained within the same file(s);
they are only separated for easy viewing.
##### Examples:
- [Formats](https://github.com/go-playground/universal-translator/tree/master/_examples/file-formats)
##### Basic Makeup
NOTE: not all fields are needed for all translation types, see [examples](https://github.com/go-playground/universal-translator/tree/master/_examples/file-formats)
```json
{
"locale": "en",
"key": "days-left",
"trans": "You have {0} day left.",
"type": "Cardinal",
"rule": "One",
"override": false
}
```
|Field|Description|
|---|---|
|locale|The locale for which the translation is for.|
|key|The translation key that will be used to store and lookup each translation; normally it is a string or integer.|
|trans|The actual translation text.|
|type|The type of translation Cardinal, Ordinal, Range or "" for a plain substitution(not required to be defined if plain used)|
|rule|The plural rule for which the translation is for eg. One, Two, Few, Many or Other.(not required to be defined if plain used)|
|override|If you wish to override an existing translation that has already been registered, set this to 'true'. 99% of the time there is no need to define it.|
Help With Tests
---------------
To anyone interesting in helping or contributing, I sure could use some help creating tests for each language.
Please see issue [here](https://github.com/go-playground/locales/issues/1) for details.
License
------
Distributed under MIT License, please see license file in code for more details.
+148
View File
@@ -0,0 +1,148 @@
package ut
import (
"errors"
"fmt"
"github.com/go-playground/locales"
)
var (
// ErrUnknowTranslation indicates the translation could not be found
ErrUnknowTranslation = errors.New("Unknown Translation")
)
var _ error = new(ErrConflictingTranslation)
var _ error = new(ErrRangeTranslation)
var _ error = new(ErrOrdinalTranslation)
var _ error = new(ErrCardinalTranslation)
var _ error = new(ErrMissingPluralTranslation)
var _ error = new(ErrExistingTranslator)
// ErrExistingTranslator is the error representing a conflicting translator
type ErrExistingTranslator struct {
locale string
}
// Error returns ErrExistingTranslator's internal error text
func (e *ErrExistingTranslator) Error() string {
return fmt.Sprintf("error: conflicting translator for locale '%s'", e.locale)
}
// ErrConflictingTranslation is the error representing a conflicting translation
type ErrConflictingTranslation struct {
locale string
key interface{}
rule locales.PluralRule
text string
}
// Error returns ErrConflictingTranslation's internal error text
func (e *ErrConflictingTranslation) Error() string {
if _, ok := e.key.(string); !ok {
return fmt.Sprintf("error: conflicting key '%#v' rule '%s' with text '%s' for locale '%s', value being ignored", e.key, e.rule, e.text, e.locale)
}
return fmt.Sprintf("error: conflicting key '%s' rule '%s' with text '%s' for locale '%s', value being ignored", e.key, e.rule, e.text, e.locale)
}
// ErrRangeTranslation is the error representing a range translation error
type ErrRangeTranslation struct {
text string
}
// Error returns ErrRangeTranslation's internal error text
func (e *ErrRangeTranslation) Error() string {
return e.text
}
// ErrOrdinalTranslation is the error representing an ordinal translation error
type ErrOrdinalTranslation struct {
text string
}
// Error returns ErrOrdinalTranslation's internal error text
func (e *ErrOrdinalTranslation) Error() string {
return e.text
}
// ErrCardinalTranslation is the error representing a cardinal translation error
type ErrCardinalTranslation struct {
text string
}
// Error returns ErrCardinalTranslation's internal error text
func (e *ErrCardinalTranslation) Error() string {
return e.text
}
// ErrMissingPluralTranslation is the error signifying a missing translation given
// the locales plural rules.
type ErrMissingPluralTranslation struct {
locale string
key interface{}
rule locales.PluralRule
translationType string
}
// Error returns ErrMissingPluralTranslation's internal error text
func (e *ErrMissingPluralTranslation) Error() string {
if _, ok := e.key.(string); !ok {
return fmt.Sprintf("error: missing '%s' plural rule '%s' for translation with key '%#v' and locale '%s'", e.translationType, e.rule, e.key, e.locale)
}
return fmt.Sprintf("error: missing '%s' plural rule '%s' for translation with key '%s' and locale '%s'", e.translationType, e.rule, e.key, e.locale)
}
// ErrMissingBracket is the error representing a missing bracket in a translation
// eg. This is a {0 <-- missing ending '}'
type ErrMissingBracket struct {
locale string
key interface{}
text string
}
// Error returns ErrMissingBracket error message
func (e *ErrMissingBracket) Error() string {
return fmt.Sprintf("error: missing bracket '{}', in translation. locale: '%s' key: '%v' text: '%s'", e.locale, e.key, e.text)
}
// ErrBadParamSyntax is the error representing a bad parameter definition in a translation
// eg. This is a {must-be-int}
type ErrBadParamSyntax struct {
locale string
param string
key interface{}
text string
}
// Error returns ErrBadParamSyntax error message
func (e *ErrBadParamSyntax) Error() string {
return fmt.Sprintf("error: bad parameter syntax, missing parameter '%s' in translation. locale: '%s' key: '%v' text: '%s'", e.param, e.locale, e.key, e.text)
}
// import/export errors
// ErrMissingLocale is the error representing an expected locale that could
// not be found aka locale not registered with the UniversalTranslator Instance
type ErrMissingLocale struct {
locale string
}
// Error returns ErrMissingLocale's internal error text
func (e *ErrMissingLocale) Error() string {
return fmt.Sprintf("error: locale '%s' not registered.", e.locale)
}
// ErrBadPluralDefinition is the error representing an incorrect plural definition
// usually found within translations defined within files during the import process.
type ErrBadPluralDefinition struct {
tl translation
}
// Error returns ErrBadPluralDefinition's internal error text
func (e *ErrBadPluralDefinition) Error() string {
return fmt.Sprintf("error: bad plural definition '%#v'", e.tl)
}

Some files were not shown because too many files have changed in this diff Show More