2020-1-capstone-design1 / PET_Project1

Go to a project
Toggle navigation Toggle navigation pinning
장수창
1adca7b2 1 parent 1702d872

added android test app

Expand all
Inline Side-by-side
Showing 114 changed files with 4854 additions and 0 deletions
# This file is based on https://github.com/github/gitignore/blob/master/Android.gitignore
*.iml
.idea/compiler.xml
.idea/copyright
.idea/dictionaries
.idea/gradle.xml
.idea/libraries
.idea/inspectionProfiles
.idea/misc.xml
.idea/modules.xml
.idea/runConfigurations.xml
.idea/tasks.xml
.idea/workspace.xml
.gradle
local.properties
.DS_Store
build/
gradleBuild/
*.apk
*.ap_
*.dex
*.class
bin/
gen/
out/
*.log
.navigation/
/captures
.externalNativeBuild
<component name="ProjectCodeStyleConfiguration">
<code_scheme name="Project" version="173">
<codeStyleSettings language="XML">
<indentOptions>
<option name="CONTINUATION_INDENT_SIZE" value="4" />
</indentOptions>
<arrangement>
<rules>
<section>
<rule>
<match>
<AND>
<NAME>xmlns:android</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>^$</XML_NAMESPACE>
</AND>
</match>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>xmlns:.*</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>^$</XML_NAMESPACE>
</AND>
</match>
<order>BY_NAME</order>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>.*:id</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>.*:name</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>name</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>^$</XML_NAMESPACE>
</AND>
</match>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>style</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>^$</XML_NAMESPACE>
</AND>
</match>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>.*</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>^$</XML_NAMESPACE>
</AND>
</match>
<order>BY_NAME</order>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>.*</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>http://schemas.android.com/apk/res/android</XML_NAMESPACE>
</AND>
</match>
<order>ANDROID_ATTRIBUTE_ORDER</order>
</rule>
</section>
<section>
<rule>
<match>
<AND>
<NAME>.*</NAME>
<XML_ATTRIBUTE />
<XML_NAMESPACE>.*</XML_NAMESPACE>
</AND>
</match>
<order>BY_NAME</order>
</rule>
</section>
</rules>
</arrangement>
</codeStyleSettings>
</code_scheme>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$/../../.." vcs="Git" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="org.tensorflow.demo">
<uses-permission android:name="android.permission.CAMERA" />
<uses-feature android:name="android.hardware.camera" />
<uses-feature android:name="android.hardware.camera.autofocus" />
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
<uses-permission android:name="android.permission.RECORD_AUDIO" />
<application android:allowBackup="true"
android:debuggable="true"
android:label="@string/app_name"
android:icon="@drawable/ic_launcher"
android:theme="@style/MaterialTheme">
<!-- <activity android:name="org.tensorflow.demo.ClassifierActivity"-->
<!-- android:screenOrientation="portrait"-->
<!-- android:label="@string/activity_name_classification">-->
<!-- <intent-filter>-->
<!-- <action android:name="android.intent.action.MAIN" />-->
<!-- <category android:name="android.intent.category.LAUNCHER" />-->
<!-- <category android:name="android.intent.category.LEANBACK_LAUNCHER" />-->
<!-- </intent-filter>-->
<!-- </activity>-->
<activity android:name="org.tensorflow.demo.DetectorActivity"
android:screenOrientation="portrait"
android:label="@string/activity_name_detection">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
<category android:name="android.intent.category.LEANBACK_LAUNCHER" />
</intent-filter>
</activity>
<!-- <activity android:name="org.tensorflow.demo.StylizeActivity"-->
<!-- android:screenOrientation="portrait"-->
<!-- android:label="@string/activity_name_stylize">-->
<!-- <intent-filter>-->
<!-- <action android:name="android.intent.action.MAIN" />-->
<!-- <category android:name="android.intent.category.LAUNCHER" />-->
<!-- <category android:name="android.intent.category.LEANBACK_LAUNCHER" />-->
<!-- </intent-filter>-->
<!-- </activity>-->
<!-- <activity android:name="org.tensorflow.demo.SpeechActivity"-->
<!-- android:screenOrientation="portrait"-->
<!-- android:label="@string/activity_name_speech">-->
<!-- <intent-filter>-->
<!-- <action android:name="android.intent.action.MAIN" />-->
<!-- <category android:name="android.intent.category.LAUNCHER" />-->
<!-- <category android:name="android.intent.category.LEANBACK_LAUNCHER" />-->
<!-- </intent-filter>-->
<!-- </activity>-->
</application>
</manifest>
# Description:
# TensorFlow camera demo app for Android.
load("@build_bazel_rules_android//android:rules.bzl", "android_binary")
load(
"//tensorflow:tensorflow.bzl",
"tf_copts",
)
package(
default_visibility = ["//visibility:public"],
licenses = ["notice"], # Apache 2.0
)
exports_files(["LICENSE"])
LINKER_SCRIPT = "jni/version_script.lds"
# libtensorflow_demo.so contains the native code for image colorspace conversion
# and object tracking used by the demo. It does not require TF as a dependency
# to build if STANDALONE_DEMO_LIB is defined.
# TF support for the demo is provided separately by libtensorflow_inference.so.
cc_binary(
name = "libtensorflow_demo.so",
srcs = glob([
"jni/**/*.cc",
"jni/**/*.h",
]),
copts = tf_copts(),
defines = ["STANDALONE_DEMO_LIB"],
linkopts = [
"-landroid",
"-ldl",
"-ljnigraphics",
"-llog",
"-lm",
"-z defs",
"-s",
"-Wl,--version-script,$(location {})".format(LINKER_SCRIPT),
],
linkshared = 1,
linkstatic = 1,
tags = [
"manual",
"notap",
],
deps = [
LINKER_SCRIPT,
],
)
cc_library(
name = "tensorflow_native_libs",
srcs = [
":libtensorflow_demo.so",
"//tensorflow/tools/android/inference_interface:libtensorflow_inference.so",
],
tags = [
"manual",
"notap",
],
)
android_binary(
name = "tensorflow_demo",
srcs = glob([
"src/**/*.java",
]),
# Package assets from assets dir as well as all model targets. Remove undesired models
# (and corresponding Activities in source) to reduce APK size.
assets = [
"//tensorflow/examples/android/assets:asset_files",
":external_assets",
],
assets_dir = "",
custom_package = "org.tensorflow.demo",
manifest = "AndroidManifest.xml",
resource_files = glob(["res/**"]),
tags = [
"manual",
"notap",
],
deps = [
":tensorflow_native_libs",
"//tensorflow/tools/android/inference_interface:android_tensorflow_inference_java",
],
)
# LINT.IfChange
filegroup(
name = "external_assets",
srcs = [
"@inception_v1//:model_files",
"@mobile_ssd//:model_files",
"@speech_commands//:model_files",
"@stylize//:model_files",
],
)
# LINT.ThenChange(//tensorflow/examples/android/download-models.gradle)
filegroup(
name = "java_files",
srcs = glob(["src/**/*.java"]),
)
filegroup(
name = "jni_files",
srcs = glob([
"jni/**/*.cc",
"jni/**/*.h",
]),
)
filegroup(
name = "resource_files",
srcs = glob(["res/**"]),
)
exports_files([
"AndroidManifest.xml",
])
# TensorFlow Android Camera Demo
This folder contains an example application utilizing TensorFlow for Android
devices.
## Description
The demos in this folder are designed to give straightforward samples of using
TensorFlow in mobile applications.
Inference is done using the [TensorFlow Android Inference
Interface](../../tools/android/inference_interface), which may be built
separately if you want a standalone library to drop into your existing
application. Object tracking and efficient YUV -> RGB conversion are handled by
`libtensorflow_demo.so`.
A device running Android 5.0 (API 21) or higher is required to run the demo due
to the use of the camera2 API, although the native libraries themselves can run
on API >= 14 devices.
## Current samples:
1. [TF Classify](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/android/src/org/tensorflow/demo/ClassifierActivity.java):
Uses the [Google Inception](https://arxiv.org/abs/1409.4842)
model to classify camera frames in real-time, displaying the top results
in an overlay on the camera image.
2. [TF Detect](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/android/src/org/tensorflow/demo/DetectorActivity.java):
Demonstrates an SSD-Mobilenet model trained using the
[Tensorflow Object Detection API](https://github.com/tensorflow/models/tree/master/research/object_detection/)
introduced in [Speed/accuracy trade-offs for modern convolutional object detectors](https://arxiv.org/abs/1611.10012) to
localize and track objects (from 80 categories) in the camera preview
in real-time.
3. [TF Stylize](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java):
Uses a model based on [A Learned Representation For Artistic
Style](https://arxiv.org/abs/1610.07629) to restyle the camera preview
image to that of a number of different artists.
4. [TF
Speech](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/android/src/org/tensorflow/demo/SpeechActivity.java):
Runs a simple speech recognition model built by the [audio training
tutorial](https://www.tensorflow.org/versions/master/tutorials/audio_recognition). Listens
for a small set of words, and highlights them in the UI when they are
recognized.
<img src="sample_images/classify1.jpg" width="30%"><img src="sample_images/stylize1.jpg" width="30%"><img src="sample_images/detect1.jpg" width="30%">
## Prebuilt Components:
The fastest path to trying the demo is to download the [prebuilt demo APK](https://storage.googleapis.com/download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
Also available are precompiled native libraries, and a jcenter package that you
may simply drop into your own applications. See
[tensorflow/tools/android/inference_interface/README.md](../../tools/android/inference_interface/README.md)
for more details.
## Running the Demo
Once the app is installed it can be started via the "TF Classify", "TF Detect",
"TF Stylize", and "TF Speech" icons, which have the orange TensorFlow logo as
their icon.
While running the activities, pressing the volume keys on your device will
toggle debug visualizations on/off, rendering additional info to the screen that
may be useful for development purposes.
## Building in Android Studio using the TensorFlow AAR from JCenter
The simplest way to compile the demo app yourself, and try out changes to the
project code is to use AndroidStudio. Simply set this `android` directory as the
project root.
Then edit the `build.gradle` file and change the value of `nativeBuildSystem` to
`'none'` so that the project is built in the simplest way possible:
```None
def nativeBuildSystem = 'none'
```
While this project includes full build integration for TensorFlow, this setting
disables it, and uses the TensorFlow Inference Interface package from JCenter.
Note: Currently, in this build mode, YUV -> RGB is done using a less efficient
Java implementation, and object tracking is not available in the "TF Detect"
activity. Setting the build system to `'cmake'` currently only builds
`libtensorflow_demo.so`, which provides fast YUV -> RGB conversion and object
tracking, while still acquiring TensorFlow support via the downloaded AAR, so it
may be a lightweight way to enable these features.
For any project that does not include custom low level TensorFlow code, this is
likely sufficient.
For details on how to include this JCenter package in your own project see
[tensorflow/tools/android/inference_interface/README.md](../../tools/android/inference_interface/README.md)
## Building the Demo with TensorFlow from Source
Pick your preferred approach below. At the moment, we have full support for
Bazel, and partial support for gradle, cmake, make, and Android Studio.
As a first step for all build types, clone the TensorFlow repo with:
```
git clone --recurse-submodules https://github.com/tensorflow/tensorflow.git
```
Note that `--recurse-submodules` is necessary to prevent some issues with
protobuf compilation.
### Bazel
NOTE: Bazel does not currently support building for Android on Windows. Full
support for gradle/cmake builds is coming soon, but in the meantime we suggest
that Windows users download the
[prebuilt demo APK](https://storage.googleapis.com/download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk)
instead.
##### Install Bazel and Android Prerequisites
Bazel is the primary build system for TensorFlow. To build with Bazel, it and
the Android NDK and SDK must be installed on your system.
1. Install the latest version of Bazel as per the instructions [on the Bazel
website](https://bazel.build/versions/master/docs/install.html).
2. The Android NDK is required to build the native (C/C++) TensorFlow code. The
current recommended version is 14b, which may be found
[here](https://developer.android.com/ndk/downloads/older_releases.html#ndk-14b-downloads).
3. The Android SDK and build tools may be obtained
[here](https://developer.android.com/tools/revisions/build-tools.html), or
alternatively as part of [Android
Studio](https://developer.android.com/studio/index.html). Build tools API >=
23 is required to build the TF Android demo (though it will run on API >= 21
devices).
##### Edit WORKSPACE
NOTE: As long as you have the SDK and NDK installed, the `./configure` script
will create these rules for you. Answer "Yes" when the script asks to
automatically configure the `./WORKSPACE`.
The Android entries in
[`<workspace_root>/WORKSPACE`](../../../WORKSPACE#L19-L36) must be uncommented
with the paths filled in appropriately depending on where you installed the NDK
and SDK. Otherwise an error such as: "The external label
'//external:android/sdk' is not bound to anything" will be reported.
Also edit the API levels for the SDK in WORKSPACE to the highest level you have
installed in your SDK. This must be >= 23 (this is completely independent of the
API level of the demo, which is defined in AndroidManifest.xml). The NDK API
level may remain at 14.
##### Install Model Files (optional)
The TensorFlow `GraphDef`s that contain the model definitions and weights are
not packaged in the repo because of their size. They are downloaded
automatically and packaged with the APK by Bazel via a new_http_archive defined
in `WORKSPACE` during the build process, and by Gradle via
download-models.gradle.
**Optional**: If you wish to place the models in your assets manually, remove
all of the `model_files` entries from the `assets` list in `tensorflow_demo`
found in the [`BUILD`](BUILD#L92) file. Then download and extract the archives
yourself to the `assets` directory in the source tree:
```bash
BASE_URL=https://storage.googleapis.com/download.tensorflow.org/models
for MODEL_ZIP in inception5h.zip ssd_mobilenet_v1_android_export.zip stylize_v1.zip
do
curl -L ${BASE_URL}/${MODEL_ZIP} -o /tmp/${MODEL_ZIP}
unzip /tmp/${MODEL_ZIP} -d tensorflow/examples/android/assets/
done
```
This will extract the models and their associated metadata files to the local
assets/ directory.
If you are using Gradle, make sure to remove download-models.gradle reference
from build.gradle after your manually download models; otherwise gradle might
download models again and overwrite your models.
##### Build
After editing your WORKSPACE file to update the SDK/NDK configuration, you may
build the APK. Run this from your workspace root:
```bash
bazel build --cxxopt='--std=c++11' -c opt //tensorflow/examples/android:tensorflow_demo
```
##### Install
Make sure that adb debugging is enabled on your Android 5.0 (API 21) or later
device, then after building use the following command from your workspace root
to install the APK:
```bash
adb install -r bazel-bin/tensorflow/examples/android/tensorflow_demo.apk
```
### Android Studio with Bazel
Android Studio may be used to build the demo in conjunction with Bazel. First,
make sure that you can build with Bazel following the above directions. Then,
look at [build.gradle](build.gradle) and make sure that the path to Bazel
matches that of your system.
At this point you can add the tensorflow/examples/android directory as a new
Android Studio project. Click through installing all the Gradle extensions it
requests, and you should be able to have Android Studio build the demo like any
other application (it will call out to Bazel to build the native code with the
NDK).
### CMake
Full CMake support for the demo is coming soon, but for now it is possible to
build the TensorFlow Android Inference library using
[tensorflow/tools/android/inference_interface/cmake](../../tools/android/inference_interface/cmake).
package(
default_visibility = ["//visibility:public"],
licenses = ["notice"], # Apache 2.0
)
# It is necessary to use this filegroup rather than globbing the files in this
# folder directly the examples/android:tensorflow_demo target due to the fact
# that assets_dir is necessarily set to "" there (to allow using other
# arbitrary targets as assets).
filegroup(
name = "asset_files",
srcs = glob(
["**/*"],
exclude = ["BUILD"],
),
)
This file is too large to display.
// This file provides basic support for building the TensorFlow demo
// in Android Studio with Gradle.
//
// Note that Bazel is still used by default to compile the native libs,
// and should be installed at the location noted below. This build file
// automates the process of calling out to it and copying the compiled
// libraries back into the appropriate directory.
//
// Alternatively, experimental support for Makefile builds is provided by
// setting nativeBuildSystem below to 'makefile'. This will allow building the demo
// on Windows machines, but note that full equivalence with the Bazel
// build is not yet guaranteed. See comments below for caveats and tips
// for speeding up the build, such as enabling ccache.
// NOTE: Running a make build will cause subsequent Bazel builds to *fail*
// unless the contrib/makefile/downloads/ and gen/ dirs are deleted afterwards.
// The cmake build only creates libtensorflow_demo.so. In this situation,
// libtensorflow_inference.so will be acquired via the tensorflow.aar dependency.
// It is necessary to customize Gradle's build directory, as otherwise
// it will conflict with the BUILD file used by Bazel on case-insensitive OSs.
project.buildDir = 'gradleBuild'
getProject().setBuildDir('gradleBuild')
buildscript {
repositories {
jcenter()
google()
}
dependencies {
classpath 'com.android.tools.build:gradle:3.3.1'
classpath 'org.apache.httpcomponents:httpclient:4.5.4'
}
}
allprojects {
repositories {
jcenter()
google()
}
}
// set to 'bazel', 'cmake', 'makefile', 'none'
def nativeBuildSystem = 'none'
// Controls output directory in APK and CPU type for Bazel builds.
// NOTE: Does not affect the Makefile build target API (yet), which currently
// assumes armeabi-v7a. If building with make, changing this will require
// editing the Makefile as well.
// The CMake build has only been tested with armeabi-v7a; others may not work.
def cpuType = 'armeabi-v7a'
// Output directory in the local directory for packaging into the APK.
def nativeOutDir = 'libs/' + cpuType
// Default to building with Bazel and override with make if requested.
def nativeBuildRule = 'buildNativeBazel'
def demoLibPath = '../../../bazel-bin/tensorflow/examples/android/libtensorflow_demo.so'
def inferenceLibPath = '../../../bazel-bin/tensorflow/tools/android/inference_interface/libtensorflow_inference.so'
// Override for Makefile builds.
if (nativeBuildSystem == 'makefile') {
nativeBuildRule = 'buildNativeMake'
demoLibPath = '../../../tensorflow/contrib/makefile/gen/lib/android_' + cpuType + '/libtensorflow_demo.so'
inferenceLibPath = '../../../tensorflow/contrib/makefile/gen/lib/android_' + cpuType + '/libtensorflow_inference.so'
}
// If building with Bazel, this is the location of the bazel binary.
// NOTE: Bazel does not yet support building for Android on Windows,
// so in this case the Makefile build must be used as described above.
def bazelLocation = '/usr/local/bin/bazel'
// import DownloadModels task
project.ext.ASSET_DIR = projectDir.toString() + '/assets'
project.ext.TMP_DIR = project.buildDir.toString() + '/downloads'
// Download default models; if you wish to use your own models then
// place them in the "assets" directory and comment out this line.
apply from: "download-models.gradle"
apply plugin: 'com.android.application'
android {
compileSdkVersion 23
if (nativeBuildSystem == 'cmake') {
defaultConfig {
applicationId = 'org.tensorflow.demo'
minSdkVersion 21
targetSdkVersion 23
ndk {
abiFilters "${cpuType}"
}
externalNativeBuild {
cmake {
arguments '-DANDROID_STL=c++_static'
}
}
}
externalNativeBuild {
cmake {
path './jni/CMakeLists.txt'
}
}
}
lintOptions {
abortOnError false
}
sourceSets {
main {
if (nativeBuildSystem == 'bazel' || nativeBuildSystem == 'makefile') {
// TensorFlow Java API sources.
java {
srcDir '../../java/src/main/java'
exclude '**/examples/**'
}
// Android TensorFlow wrappers, etc.
java {
srcDir '../../tools/android/inference_interface/java'
}
}
// Android demo app sources.
java {
srcDir 'src'
}
manifest.srcFile 'AndroidManifest.xml'
resources.srcDirs = ['src']
aidl.srcDirs = ['src']
renderscript.srcDirs = ['src']
res.srcDirs = ['res']
assets.srcDirs = [project.ext.ASSET_DIR]
jniLibs.srcDirs = ['libs']
}
debug.setRoot('build-types/debug')
release.setRoot('build-types/release')
}
defaultConfig {
targetSdkVersion 23
minSdkVersion 21
}
}
task buildNativeBazel(type: Exec) {
workingDir '../../..'
commandLine bazelLocation, 'build', '-c', 'opt', \
'tensorflow/examples/android:tensorflow_native_libs', \
'--crosstool_top=//external:android/crosstool', \
'--cpu=' + cpuType, \
'--host_crosstool_top=@bazel_tools//tools/cpp:toolchain'
}
task buildNativeMake(type: Exec) {
environment "NDK_ROOT", android.ndkDirectory
// Tip: install ccache and uncomment the following to speed up
// builds significantly.
// environment "CC_PREFIX", 'ccache'
workingDir '../../..'
commandLine 'tensorflow/contrib/makefile/build_all_android.sh', \
'-s', \
'tensorflow/contrib/makefile/sub_makefiles/android/Makefile.in', \
'-t', \
'libtensorflow_inference.so libtensorflow_demo.so all' \
, '-a', cpuType \
//, '-T' // Uncomment to skip protobuf and speed up subsequent builds.
}
task copyNativeLibs(type: Copy) {
from demoLibPath
from inferenceLibPath
into nativeOutDir
duplicatesStrategy = 'include'
dependsOn nativeBuildRule
fileMode 0644
}
tasks.whenTaskAdded { task ->
if (nativeBuildSystem == 'bazel' || nativeBuildSystem == 'makefile') {
if (task.name == 'assembleDebug') {
task.dependsOn 'copyNativeLibs'
}
if (task.name == 'assembleRelease') {
task.dependsOn 'copyNativeLibs'
}
}
}
dependencies {
if (nativeBuildSystem == 'cmake' || nativeBuildSystem == 'none') {
implementation 'org.tensorflow:tensorflow-android:+'
}
}
/*
* download-models.gradle
* Downloads model files from ${MODEL_URL} into application's asset folder
* Input:
* project.ext.TMP_DIR: absolute path to hold downloaded zip files
* project.ext.ASSET_DIR: absolute path to save unzipped model files
* Output:
* 3 model files will be downloaded into given folder of ext.ASSET_DIR
*/
// hard coded model files
// LINT.IfChange
def models = ['inception_v1.zip',
'object_detection/ssd_mobilenet_v1_android_export.zip',
'stylize_v1.zip',
'speech_commands_conv_actions.zip']
// LINT.ThenChange(//tensorflow/examples/android/BUILD)
// Root URL for model archives
def MODEL_URL = 'https://storage.googleapis.com/download.tensorflow.org/models'
buildscript {
repositories {
jcenter()
}
dependencies {
classpath 'de.undercouch:gradle-download-task:3.2.0'
}
}
import de.undercouch.gradle.tasks.download.Download
task downloadFile(type: Download){
for (f in models) {
src "${MODEL_URL}/" + f
}
dest new File(project.ext.TMP_DIR)
overwrite true
}
task extractModels(type: Copy) {
for (f in models) {
def localFile = f.split("/")[-1]
from zipTree(project.ext.TMP_DIR + '/' + localFile)
}
into file(project.ext.ASSET_DIR)
fileMode 0644
exclude '**/LICENSE'
def needDownload = false
for (f in models) {
def localFile = f.split("/")[-1]
if (!(new File(project.ext.TMP_DIR + '/' + localFile)).exists()) {
needDownload = true
}
}
if (needDownload) {
dependsOn downloadFile
}
}
tasks.whenTaskAdded { task ->
if (task.name == 'assembleDebug') {
task.dependsOn 'extractModels'
}
if (task.name == 'assembleRelease') {
task.dependsOn 'extractModels'
}
}
#Sat Nov 18 15:06:47 CET 2017
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-all.zip
#!/usr/bin/env bash
##############################################################################
##
## Gradle start up script for UN*X
##
##############################################################################
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS=""
APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
warn ( ) {
echo "$*"
}
die ( ) {
echo
echo "$*"
echo
exit 1
}
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
case "`uname`" in
CYGWIN* )
cygwin=true
;;
Darwin* )
darwin=true
;;
MINGW* )
msys=true
;;
esac
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
PRG="$0"
# Need this for relative symlinks.
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`"/$link"
fi
done
SAVED="`pwd`"
cd "`dirname \"$PRG\"`/" >/dev/null
APP_HOME="`pwd -P`"
cd "$SAVED" >/dev/null
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD="java"
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
MAX_FD_LIMIT=`ulimit -H -n`
if [ $? -eq 0 ] ; then
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
MAX_FD="$MAX_FD_LIMIT"
fi
ulimit -n $MAX_FD
if [ $? -ne 0 ] ; then
warn "Could not set maximum file descriptor limit: $MAX_FD"
fi
else
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
fi
fi
# For Darwin, add options to specify how the application appears in the dock
if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi
# For Cygwin, switch paths to Windows format before running java
if $cygwin ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"`
# We build the pattern for arguments to be converted via cygpath
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
SEP=""
for dir in $ROOTDIRSRAW ; do
ROOTDIRS="$ROOTDIRS$SEP$dir"
SEP="|"
done
OURCYGPATTERN="(^($ROOTDIRS))"
# Add a user-defined pattern to the cygpath arguments
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
fi
# Now convert the arguments - kludge to limit ourselves to /bin/sh
i=0
for arg in "$@" ; do
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
else
eval `echo args$i`="\"$arg\""
fi
i=$((i+1))
done
case $i in
(0) set -- ;;
(1) set -- "$args0" ;;
(2) set -- "$args0" "$args1" ;;
(3) set -- "$args0" "$args1" "$args2" ;;
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
function splitJvmOpts() {
JVM_OPTS=("$@")
}
eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS=
set DIRNAME=%~dp0
if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if "%ERRORLEVEL%" == "0" goto init
echo.
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto init
echo.
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:init
@rem Get command-line arguments, handling Windowz variants
if not "%OS%" == "Windows_NT" goto win9xME_args
if "%@eval[2+2]" == "4" goto 4NT_args
:win9xME_args
@rem Slurp the command line arguments.
set CMD_LINE_ARGS=
set _SKIP=2
:win9xME_args_slurp
if "x%~1" == "x" goto execute
set CMD_LINE_ARGS=%*
goto execute
:4NT_args
@rem Get arguments from the 4NT Shell from JP Software
set CMD_LINE_ARGS=%$
:execute
@rem Setup the command line
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
:end
@rem End local scope for the variables with windows NT shell
if "%ERRORLEVEL%"=="0" goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
exit /b 1
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega
#
# Copyright (C) 2016 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
project(TENSORFLOW_DEMO)
cmake_minimum_required(VERSION 3.4.1)
set(CMAKE_VERBOSE_MAKEFILE on)
get_filename_component(TF_SRC_ROOT ${CMAKE_SOURCE_DIR}/../../../.. ABSOLUTE)
get_filename_component(SAMPLE_SRC_DIR ${CMAKE_SOURCE_DIR}/.. ABSOLUTE)
if (ANDROID_ABI MATCHES "^armeabi-v7a$")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfloat-abi=softfp -mfpu=neon")
elseif(ANDROID_ABI MATCHES "^arm64-v8a")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -ftree-vectorize")
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSTANDALONE_DEMO_LIB \
-std=c++11 -fno-exceptions -fno-rtti -O2 -Wno-narrowing \
-fPIE")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} \
-Wl,--allow-multiple-definition \
-Wl,--whole-archive -fPIE -v")
file(GLOB_RECURSE tensorflow_demo_sources ${SAMPLE_SRC_DIR}/jni/*.*)
add_library(tensorflow_demo SHARED
${tensorflow_demo_sources})
target_include_directories(tensorflow_demo PRIVATE
${TF_SRC_ROOT}
${CMAKE_SOURCE_DIR})
target_link_libraries(tensorflow_demo
android
log
jnigraphics
m
atomic
z)
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This file binds the native image utility code to the Java class
// which exposes them.
#include <jni.h>
#include <stdio.h>
#include <stdlib.h>
#include "tensorflow/examples/android/jni/rgb2yuv.h"
#include "tensorflow/examples/android/jni/yuv2rgb.h"
#define IMAGEUTILS_METHOD(METHOD_NAME) \
Java_org_tensorflow_demo_env_ImageUtils_##METHOD_NAME // NOLINT
#ifdef __cplusplus
extern "C" {
#endif
JNIEXPORT void JNICALL
IMAGEUTILS_METHOD(convertYUV420SPToARGB8888)(
JNIEnv* env, jclass clazz, jbyteArray input, jintArray output,
jint width, jint height, jboolean halfSize);
JNIEXPORT void JNICALL IMAGEUTILS_METHOD(convertYUV420ToARGB8888)(
JNIEnv* env, jclass clazz, jbyteArray y, jbyteArray u, jbyteArray v,
jintArray output, jint width, jint height, jint y_row_stride,
jint uv_row_stride, jint uv_pixel_stride, jboolean halfSize);
JNIEXPORT void JNICALL IMAGEUTILS_METHOD(convertYUV420SPToRGB565)(
JNIEnv* env, jclass clazz, jbyteArray input, jbyteArray output, jint width,
jint height);
JNIEXPORT void JNICALL
IMAGEUTILS_METHOD(convertARGB8888ToYUV420SP)(
JNIEnv* env, jclass clazz, jintArray input, jbyteArray output,
jint width, jint height);
JNIEXPORT void JNICALL
IMAGEUTILS_METHOD(convertRGB565ToYUV420SP)(
JNIEnv* env, jclass clazz, jbyteArray input, jbyteArray output,
jint width, jint height);
#ifdef __cplusplus
}
#endif
JNIEXPORT void JNICALL
IMAGEUTILS_METHOD(convertYUV420SPToARGB8888)(
JNIEnv* env, jclass clazz, jbyteArray input, jintArray output,
jint width, jint height, jboolean halfSize) {
jboolean inputCopy = JNI_FALSE;
jbyte* const i = env->GetByteArrayElements(input, &inputCopy);
jboolean outputCopy = JNI_FALSE;
jint* const o = env->GetIntArrayElements(output, &outputCopy);
if (halfSize) {
ConvertYUV420SPToARGB8888HalfSize(reinterpret_cast<uint8_t*>(i),
reinterpret_cast<uint32_t*>(o), width,
height);
} else {
ConvertYUV420SPToARGB8888(reinterpret_cast<uint8_t*>(i),
reinterpret_cast<uint8_t*>(i) + width * height,
reinterpret_cast<uint32_t*>(o), width, height);
}
env->ReleaseByteArrayElements(input, i, JNI_ABORT);
env->ReleaseIntArrayElements(output, o, 0);
}
JNIEXPORT void JNICALL IMAGEUTILS_METHOD(convertYUV420ToARGB8888)(
JNIEnv* env, jclass clazz, jbyteArray y, jbyteArray u, jbyteArray v,
jintArray output, jint width, jint height, jint y_row_stride,
jint uv_row_stride, jint uv_pixel_stride, jboolean halfSize) {
jboolean inputCopy = JNI_FALSE;
jbyte* const y_buff = env->GetByteArrayElements(y, &inputCopy);
jboolean outputCopy = JNI_FALSE;
jint* const o = env->GetIntArrayElements(output, &outputCopy);
if (halfSize) {
ConvertYUV420SPToARGB8888HalfSize(reinterpret_cast<uint8_t*>(y_buff),
reinterpret_cast<uint32_t*>(o), width,
height);
} else {
jbyte* const u_buff = env->GetByteArrayElements(u, &inputCopy);
jbyte* const v_buff = env->GetByteArrayElements(v, &inputCopy);
ConvertYUV420ToARGB8888(
reinterpret_cast<uint8_t*>(y_buff), reinterpret_cast<uint8_t*>(u_buff),
reinterpret_cast<uint8_t*>(v_buff), reinterpret_cast<uint32_t*>(o),
width, height, y_row_stride, uv_row_stride, uv_pixel_stride);
env->ReleaseByteArrayElements(u, u_buff, JNI_ABORT);
env->ReleaseByteArrayElements(v, v_buff, JNI_ABORT);
}
env->ReleaseByteArrayElements(y, y_buff, JNI_ABORT);
env->ReleaseIntArrayElements(output, o, 0);
}
JNIEXPORT void JNICALL IMAGEUTILS_METHOD(convertYUV420SPToRGB565)(
JNIEnv* env, jclass clazz, jbyteArray input, jbyteArray output, jint width,
jint height) {
jboolean inputCopy = JNI_FALSE;
jbyte* const i = env->GetByteArrayElements(input, &inputCopy);
jboolean outputCopy = JNI_FALSE;
jbyte* const o = env->GetByteArrayElements(output, &outputCopy);
ConvertYUV420SPToRGB565(reinterpret_cast<uint8_t*>(i),
reinterpret_cast<uint16_t*>(o), width, height);
env->ReleaseByteArrayElements(input, i, JNI_ABORT);
env->ReleaseByteArrayElements(output, o, 0);
}
JNIEXPORT void JNICALL
IMAGEUTILS_METHOD(convertARGB8888ToYUV420SP)(
JNIEnv* env, jclass clazz, jintArray input, jbyteArray output,
jint width, jint height) {
jboolean inputCopy = JNI_FALSE;
jint* const i = env->GetIntArrayElements(input, &inputCopy);
jboolean outputCopy = JNI_FALSE;
jbyte* const o = env->GetByteArrayElements(output, &outputCopy);
ConvertARGB8888ToYUV420SP(reinterpret_cast<uint32_t*>(i),
reinterpret_cast<uint8_t*>(o), width, height);
env->ReleaseIntArrayElements(input, i, JNI_ABORT);
env->ReleaseByteArrayElements(output, o, 0);
}
JNIEXPORT void JNICALL
IMAGEUTILS_METHOD(convertRGB565ToYUV420SP)(
JNIEnv* env, jclass clazz, jbyteArray input, jbyteArray output,
jint width, jint height) {
jboolean inputCopy = JNI_FALSE;
jbyte* const i = env->GetByteArrayElements(input, &inputCopy);
jboolean outputCopy = JNI_FALSE;
jbyte* const o = env->GetByteArrayElements(output, &outputCopy);
ConvertRGB565ToYUV420SP(reinterpret_cast<uint16_t*>(i),
reinterpret_cast<uint8_t*>(o), width, height);
env->ReleaseByteArrayElements(input, i, JNI_ABORT);
env->ReleaseByteArrayElements(output, o, 0);
}
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_FRAME_PAIR_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_FRAME_PAIR_H_
#include "tensorflow/examples/android/jni/object_tracking/keypoint.h"
namespace tf_tracking {
// A class that records keypoint correspondences from pairs of
// consecutive frames.
class FramePair {
public:
FramePair()
: start_time_(0),
end_time_(0),
number_of_keypoints_(0) {}
// Cleans up the FramePair so that they can be reused.
void Init(const int64_t start_time, const int64_t end_time);
void AdjustBox(const BoundingBox box,
float* const translation_x,
float* const translation_y,
float* const scale_x,
float* const scale_y) const;
private:
// Returns the weighted median of the given deltas, computed independently on
// x and y. Returns 0,0 in case of failure. The assumption is that a
// translation of 0.0 in the degenerate case is the best that can be done, and
// should not be considered an error.
//
// In the case of scale, a slight exception is made just to be safe and
// there is a check for 0.0 explicitly, but that shouldn't ever be possible to
// happen naturally because of the non-zero + parity checks in FillScales.
Point2f GetWeightedMedian(const float* const weights,
const Point2f* const deltas) const;
float GetWeightedMedianScale(const float* const weights,
const Point2f* const deltas) const;
// Weights points based on the query_point and cutoff_dist.
int FillWeights(const BoundingBox& box,
float* const weights) const;
// Fills in the array of deltas with the translations of the points
// between frames.
void FillTranslations(Point2f* const translations) const;
// Fills in the array of deltas with the relative scale factor of points
// relative to a given center. Has the ability to override the weight to 0 if
// a degenerate scale is detected.
// Translation is the amount the center of the box has moved from one frame to
// the next.
int FillScales(const Point2f& old_center,
const Point2f& translation,
float* const weights,
Point2f* const scales) const;
// TODO(andrewharp): Make these private.
public:
// The time at frame1.
int64_t start_time_;
// The time at frame2.
int64_t end_time_;
// This array will contain the keypoints found in frame 1.
Keypoint frame1_keypoints_[kMaxKeypoints];
// Contain the locations of the keypoints from frame 1 in frame 2.
Keypoint frame2_keypoints_[kMaxKeypoints];
// The number of keypoints in frame 1.
int number_of_keypoints_;
// Keeps track of which keypoint correspondences were actually found from one
// frame to another.
// The i-th element of this array will be non-zero if and only if the i-th
// keypoint of frame 1 was found in frame 2.
bool optical_flow_found_keypoint_[kMaxKeypoints];
private:
TF_DISALLOW_COPY_AND_ASSIGN(FramePair);
};
} // namespace tf_tracking
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_FRAME_PAIR_H_
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_GEOM_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_GEOM_H_
#include "tensorflow/examples/android/jni/object_tracking/logging.h"
#include "tensorflow/examples/android/jni/object_tracking/utils.h"
namespace tf_tracking {
struct Size {
Size(const int width, const int height) : width(width), height(height) {}
int width;
int height;
};
class Point2f {
public:
Point2f() : x(0.0f), y(0.0f) {}
Point2f(const float x, const float y) : x(x), y(y) {}
inline Point2f operator- (const Point2f& that) const {
return Point2f(this->x - that.x, this->y - that.y);
}
inline Point2f operator+ (const Point2f& that) const {
return Point2f(this->x + that.x, this->y + that.y);
}
inline Point2f& operator+= (const Point2f& that) {
this->x += that.x;
this->y += that.y;
return *this;
}
inline Point2f& operator-= (const Point2f& that) {
this->x -= that.x;
this->y -= that.y;
return *this;
}
inline Point2f operator- (const Point2f& that) {
return Point2f(this->x - that.x, this->y - that.y);
}
inline float LengthSquared() {
return Square(this->x) + Square(this->y);
}
inline float Length() {
return sqrtf(LengthSquared());
}
inline float DistanceSquared(const Point2f& that) {
return Square(this->x - that.x) + Square(this->y - that.y);
}
inline float Distance(const Point2f& that) {
return sqrtf(DistanceSquared(that));
}
float x;
float y;
};
inline std::ostream& operator<<(std::ostream& stream, const Point2f& point) {
stream << point.x << "," << point.y;
return stream;
}
class BoundingBox {
public:
BoundingBox()
: left_(0),
top_(0),
right_(0),
bottom_(0) {}
BoundingBox(const BoundingBox& bounding_box)
: left_(bounding_box.left_),
top_(bounding_box.top_),
right_(bounding_box.right_),
bottom_(bounding_box.bottom_) {
SCHECK(left_ < right_, "Bounds out of whack! %.2f vs %.2f!", left_, right_);
SCHECK(top_ < bottom_, "Bounds out of whack! %.2f vs %.2f!", top_, bottom_);
}
BoundingBox(const float left,
const float top,
const float right,
const float bottom)
: left_(left),
top_(top),
right_(right),
bottom_(bottom) {
SCHECK(left_ < right_, "Bounds out of whack! %.2f vs %.2f!", left_, right_);
SCHECK(top_ < bottom_, "Bounds out of whack! %.2f vs %.2f!", top_, bottom_);
}
BoundingBox(const Point2f& point1, const Point2f& point2)
: left_(MIN(point1.x, point2.x)),
top_(MIN(point1.y, point2.y)),
right_(MAX(point1.x, point2.x)),
bottom_(MAX(point1.y, point2.y)) {}
inline void CopyToArray(float* const bounds_array) const {
bounds_array[0] = left_;
bounds_array[1] = top_;
bounds_array[2] = right_;
bounds_array[3] = bottom_;
}
inline float GetWidth() const {
return right_ - left_;
}
inline float GetHeight() const {
return bottom_ - top_;
}
inline float GetArea() const {
const float width = GetWidth();
const float height = GetHeight();
if (width <= 0 || height <= 0) {
return 0.0f;
}
return width * height;
}
inline Point2f GetCenter() const {
return Point2f((left_ + right_) / 2.0f,
(top_ + bottom_) / 2.0f);
}
inline bool ValidBox() const {
return GetArea() > 0.0f;
}
// Returns a bounding box created from the overlapping area of these two.
inline BoundingBox Intersect(const BoundingBox& that) const {
const float new_left = MAX(this->left_, that.left_);
const float new_right = MIN(this->right_, that.right_);
if (new_left >= new_right) {
return BoundingBox();
}
const float new_top = MAX(this->top_, that.top_);
const float new_bottom = MIN(this->bottom_, that.bottom_);
if (new_top >= new_bottom) {
return BoundingBox();
}
return BoundingBox(new_left, new_top, new_right, new_bottom);
}
// Returns a bounding box that can contain both boxes.
inline BoundingBox Union(const BoundingBox& that) const {
return BoundingBox(MIN(this->left_, that.left_),
MIN(this->top_, that.top_),
MAX(this->right_, that.right_),
MAX(this->bottom_, that.bottom_));
}
inline float PascalScore(const BoundingBox& that) const {
SCHECK(GetArea() > 0.0f, "Empty bounding box!");
SCHECK(that.GetArea() > 0.0f, "Empty bounding box!");
const float intersect_area = this->Intersect(that).GetArea();
if (intersect_area <= 0) {
return 0;
}
const float score =
intersect_area / (GetArea() + that.GetArea() - intersect_area);
SCHECK(InRange(score, 0.0f, 1.0f), "Invalid score! %.2f", score);
return score;
}
inline bool Intersects(const BoundingBox& that) const {
return InRange(that.left_, left_, right_)
|| InRange(that.right_, left_, right_)
|| InRange(that.top_, top_, bottom_)
|| InRange(that.bottom_, top_, bottom_);
}
// Returns whether another bounding box is completely inside of this bounding
// box. Sharing edges is ok.
inline bool Contains(const BoundingBox& that) const {
return that.left_ >= left_ &&
that.right_ <= right_ &&
that.top_ >= top_ &&
that.bottom_ <= bottom_;
}
inline bool Contains(const Point2f& point) const {
return InRange(point.x, left_, right_) && InRange(point.y, top_, bottom_);
}
inline void Shift(const Point2f shift_amount) {
left_ += shift_amount.x;
top_ += shift_amount.y;
right_ += shift_amount.x;
bottom_ += shift_amount.y;
}
inline void ScaleOrigin(const float scale_x, const float scale_y) {
left_ *= scale_x;
right_ *= scale_x;
top_ *= scale_y;
bottom_ *= scale_y;
}
inline void Scale(const float scale_x, const float scale_y) {
const Point2f center = GetCenter();
const float half_width = GetWidth() / 2.0f;
const float half_height = GetHeight() / 2.0f;
left_ = center.x - half_width * scale_x;
right_ = center.x + half_width * scale_x;
top_ = center.y - half_height * scale_y;
bottom_ = center.y + half_height * scale_y;
}
float left_;
float top_;
float right_;
float bottom_;
};
inline std::ostream& operator<<(std::ostream& stream, const BoundingBox& box) {
stream << "[" << box.left_ << " - " << box.right_
<< ", " << box.top_ << " - " << box.bottom_
<< ", w:" << box.GetWidth() << " h:" << box.GetHeight() << "]";
return stream;
}
class BoundingSquare {
public:
BoundingSquare(const float x, const float y, const float size)
: x_(x), y_(y), size_(size) {}
explicit BoundingSquare(const BoundingBox& box)
: x_(box.left_), y_(box.top_), size_(box.GetWidth()) {
#ifdef SANITY_CHECKS
if (std::abs(box.GetWidth() - box.GetHeight()) > 0.1f) {
LOG(WARNING) << "This is not a square: " << box << std::endl;
}
#endif
}
inline BoundingBox ToBoundingBox() const {
return BoundingBox(x_, y_, x_ + size_, y_ + size_);
}
inline bool ValidBox() {
return size_ > 0.0f;
}
inline void Shift(const Point2f shift_amount) {
x_ += shift_amount.x;
y_ += shift_amount.y;
}
inline void Scale(const float scale) {
const float new_size = size_ * scale;
const float position_diff = (new_size - size_) / 2.0f;
x_ -= position_diff;
y_ -= position_diff;
size_ = new_size;
}
float x_;
float y_;
float size_;
};
inline std::ostream& operator<<(std::ostream& stream,
const BoundingSquare& square) {
stream << "[" << square.x_ << "," << square.y_ << " " << square.size_ << "]";
return stream;
}
inline BoundingSquare GetCenteredSquare(const BoundingBox& original_box,
const float size) {
const float width_diff = (original_box.GetWidth() - size) / 2.0f;
const float height_diff = (original_box.GetHeight() - size) / 2.0f;
return BoundingSquare(original_box.left_ + width_diff,
original_box.top_ + height_diff,
size);
}
inline BoundingSquare GetCenteredSquare(const BoundingBox& original_box) {
return GetCenteredSquare(
original_box, MIN(original_box.GetWidth(), original_box.GetHeight()));
}
} // namespace tf_tracking
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_GEOM_H_
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_GL_UTILS_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_GL_UTILS_H_
#include <GLES/gl.h>
#include <GLES/glext.h>
#include "tensorflow/examples/android/jni/object_tracking/geom.h"
namespace tf_tracking {
// Draws a box at the given position.
inline static void DrawBox(const BoundingBox& bounding_box) {
const GLfloat line[] = {
bounding_box.left_, bounding_box.bottom_,
bounding_box.left_, bounding_box.top_,
bounding_box.left_, bounding_box.top_,
bounding_box.right_, bounding_box.top_,
bounding_box.right_, bounding_box.top_,
bounding_box.right_, bounding_box.bottom_,
bounding_box.right_, bounding_box.bottom_,
bounding_box.left_, bounding_box.bottom_
};
glVertexPointer(2, GL_FLOAT, 0, line);
glEnableClientState(GL_VERTEX_ARRAY);
glDrawArrays(GL_LINES, 0, 8);
}
// Changes the coordinate system such that drawing to an arbitrary square in
// the world can thereafter be drawn to using coordinates 0 - 1.
inline static void MapWorldSquareToUnitSquare(const BoundingSquare& square) {
glScalef(square.size_, square.size_, 1.0f);
glTranslatef(square.x_ / square.size_, square.y_ / square.size_, 0.0f);
}
} // namespace tf_tracking
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_GL_UTILS_H_
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_IMAGE_DATA_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_IMAGE_DATA_H_
#include <stdint.h>
#include <memory>
#include "tensorflow/examples/android/jni/object_tracking/image-inl.h"
#include "tensorflow/examples/android/jni/object_tracking/image.h"
#include "tensorflow/examples/android/jni/object_tracking/image_utils.h"
#include "tensorflow/examples/android/jni/object_tracking/integral_image.h"
#include "tensorflow/examples/android/jni/object_tracking/time_log.h"
#include "tensorflow/examples/android/jni/object_tracking/utils.h"
#include "tensorflow/examples/android/jni/object_tracking/config.h"
namespace tf_tracking {
// Class that encapsulates all bulky processed data for a frame.
class ImageData {
public:
explicit ImageData(const int width, const int height)
: uv_frame_width_(width << 1),
uv_frame_height_(height << 1),
timestamp_(0),
image_(width, height) {
InitPyramid(width, height);
ResetComputationCache();
}
private:
void ResetComputationCache() {
uv_data_computed_ = false;
integral_image_computed_ = false;
for (int i = 0; i < kNumPyramidLevels; ++i) {
spatial_x_computed_[i] = false;
spatial_y_computed_[i] = false;
pyramid_sqrt2_computed_[i * 2] = false;
pyramid_sqrt2_computed_[i * 2 + 1] = false;
}
}
void InitPyramid(const int width, const int height) {
int level_width = width;
int level_height = height;
for (int i = 0; i < kNumPyramidLevels; ++i) {
pyramid_sqrt2_[i * 2] = NULL;
pyramid_sqrt2_[i * 2 + 1] = NULL;
spatial_x_[i] = NULL;
spatial_y_[i] = NULL;
level_width /= 2;
level_height /= 2;
}
// Alias the first pyramid level to image_.
pyramid_sqrt2_[0] = &image_;
}
public:
~ImageData() {
// The first pyramid level is actually an alias to image_,
// so make sure it doesn't get deleted here.
pyramid_sqrt2_[0] = NULL;
for (int i = 0; i < kNumPyramidLevels; ++i) {
SAFE_DELETE(pyramid_sqrt2_[i * 2]);
SAFE_DELETE(pyramid_sqrt2_[i * 2 + 1]);
SAFE_DELETE(spatial_x_[i]);
SAFE_DELETE(spatial_y_[i]);
}
}
void SetData(const uint8_t* const new_frame, const int stride,
const int64_t timestamp, const int downsample_factor) {
SetData(new_frame, NULL, stride, timestamp, downsample_factor);
}
void SetData(const uint8_t* const new_frame, const uint8_t* const uv_frame,
const int stride, const int64_t timestamp,
const int downsample_factor) {
ResetComputationCache();
timestamp_ = timestamp;
TimeLog("SetData!");
pyramid_sqrt2_[0]->FromArray(new_frame, stride, downsample_factor);
pyramid_sqrt2_computed_[0] = true;
TimeLog("Downsampled image");
if (uv_frame != NULL) {
if (u_data_.get() == NULL) {
u_data_.reset(new Image<uint8_t>(uv_frame_width_, uv_frame_height_));
v_data_.reset(new Image<uint8_t>(uv_frame_width_, uv_frame_height_));
}
GetUV(uv_frame, u_data_.get(), v_data_.get());
uv_data_computed_ = true;
TimeLog("Copied UV data");
} else {
LOGV("No uv data!");
}
#ifdef LOG_TIME
// If profiling is enabled, precompute here to make it easier to distinguish
// total costs.
Precompute();
#endif
}
inline const uint64_t GetTimestamp() const { return timestamp_; }
inline const Image<uint8_t>* GetImage() const {
SCHECK(pyramid_sqrt2_computed_[0], "image not set!");
return pyramid_sqrt2_[0];
}
const Image<uint8_t>* GetPyramidSqrt2Level(const int level) const {
if (!pyramid_sqrt2_computed_[level]) {
SCHECK(level != 0, "Level equals 0!");
if (level == 1) {
const Image<uint8_t>& upper_level = *GetPyramidSqrt2Level(0);
if (pyramid_sqrt2_[level] == NULL) {
const int new_width =
(static_cast<int>(upper_level.GetWidth() / sqrtf(2)) + 1) / 2 * 2;
const int new_height =
(static_cast<int>(upper_level.GetHeight() / sqrtf(2)) + 1) / 2 *
2;
pyramid_sqrt2_[level] = new Image<uint8_t>(new_width, new_height);
}
pyramid_sqrt2_[level]->DownsampleInterpolateLinear(upper_level);
} else {
const Image<uint8_t>& upper_level = *GetPyramidSqrt2Level(level - 2);
if (pyramid_sqrt2_[level] == NULL) {
pyramid_sqrt2_[level] = new Image<uint8_t>(
upper_level.GetWidth() / 2, upper_level.GetHeight() / 2);
}
pyramid_sqrt2_[level]->DownsampleAveraged(
upper_level.data(), upper_level.stride(), 2);
}
pyramid_sqrt2_computed_[level] = true;
}
return pyramid_sqrt2_[level];
}
inline const Image<int32_t>* GetSpatialX(const int level) const {
if (!spatial_x_computed_[level]) {
const Image<uint8_t>& src = *GetPyramidSqrt2Level(level * 2);
if (spatial_x_[level] == NULL) {
spatial_x_[level] = new Image<int32_t>(src.GetWidth(), src.GetHeight());
}
spatial_x_[level]->DerivativeX(src);
spatial_x_computed_[level] = true;
}
return spatial_x_[level];
}
inline const Image<int32_t>* GetSpatialY(const int level) const {
if (!spatial_y_computed_[level]) {
const Image<uint8_t>& src = *GetPyramidSqrt2Level(level * 2);
if (spatial_y_[level] == NULL) {
spatial_y_[level] = new Image<int32_t>(src.GetWidth(), src.GetHeight());
}
spatial_y_[level]->DerivativeY(src);
spatial_y_computed_[level] = true;
}
return spatial_y_[level];
}
// The integral image is currently only used for object detection, so lazily
// initialize it on request.
inline const IntegralImage* GetIntegralImage() const {
if (integral_image_.get() == NULL) {
integral_image_.reset(new IntegralImage(image_));
} else if (!integral_image_computed_) {
integral_image_->Recompute(image_);
}
integral_image_computed_ = true;
return integral_image_.get();
}
inline const Image<uint8_t>* GetU() const {
SCHECK(uv_data_computed_, "UV data not provided!");
return u_data_.get();
}
inline const Image<uint8_t>* GetV() const {
SCHECK(uv_data_computed_, "UV data not provided!");
return v_data_.get();
}
private:
void Precompute() {
// Create the smoothed pyramids.
for (int i = 0; i < kNumPyramidLevels * 2; i += 2) {
(void) GetPyramidSqrt2Level(i);
}
TimeLog("Created smoothed pyramids");
// Create the smoothed pyramids.
for (int i = 1; i < kNumPyramidLevels * 2; i += 2) {
(void) GetPyramidSqrt2Level(i);
}
TimeLog("Created smoothed sqrt pyramids");
// Create the spatial derivatives for frame 1.
for (int i = 0; i < kNumPyramidLevels; ++i) {
(void) GetSpatialX(i);
(void) GetSpatialY(i);
}
TimeLog("Created spatial derivatives");
(void) GetIntegralImage();
TimeLog("Got integral image!");
}
const int uv_frame_width_;
const int uv_frame_height_;
int64_t timestamp_;
Image<uint8_t> image_;
bool uv_data_computed_;
std::unique_ptr<Image<uint8_t> > u_data_;
std::unique_ptr<Image<uint8_t> > v_data_;
mutable bool spatial_x_computed_[kNumPyramidLevels];
mutable Image<int32_t>* spatial_x_[kNumPyramidLevels];
mutable bool spatial_y_computed_[kNumPyramidLevels];
mutable Image<int32_t>* spatial_y_[kNumPyramidLevels];
// Mutable so the lazy initialization can work when this class is const.
// Whether or not the integral image has been computed for the current image.
mutable bool integral_image_computed_;
mutable std::unique_ptr<IntegralImage> integral_image_;
mutable bool pyramid_sqrt2_computed_[kNumPyramidLevels * 2];
mutable Image<uint8_t>* pyramid_sqrt2_[kNumPyramidLevels * 2];
TF_DISALLOW_COPY_AND_ASSIGN(ImageData);
};
} // namespace tf_tracking
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_IMAGE_DATA_H_
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// NEON implementations of Image methods for compatible devices. Control
// should never enter this compilation unit on incompatible devices.
#ifdef __ARM_NEON
#include <arm_neon.h>
#include <stdint.h>
#include "tensorflow/examples/android/jni/object_tracking/image-inl.h"
#include "tensorflow/examples/android/jni/object_tracking/image.h"
#include "tensorflow/examples/android/jni/object_tracking/image_utils.h"
#include "tensorflow/examples/android/jni/object_tracking/utils.h"
namespace tf_tracking {
// This function does the bulk of the work.
template <>
void Image<uint8_t>::Downsample2x32ColumnsNeon(const uint8_t* const original,
const int stride,
const int orig_x) {
// Divide input x offset by 2 to find output offset.
const int new_x = orig_x >> 1;
// Initial offset into top row.
const uint8_t* offset = original + orig_x;
// This points to the leftmost pixel of our 8 horizontally arranged
// pixels in the destination data.
uint8_t* ptr_dst = (*this)[0] + new_x;
// Sum along vertical columns.
// Process 32x2 input pixels and 16x1 output pixels per iteration.
for (int new_y = 0; new_y < height_; ++new_y) {
uint16x8_t accum1 = vdupq_n_u16(0);
uint16x8_t accum2 = vdupq_n_u16(0);
// Go top to bottom across the four rows of input pixels that make up
// this output row.
for (int row_num = 0; row_num < 2; ++row_num) {
// First 16 bytes.
{
// Load 16 bytes of data from current offset.
const uint8x16_t curr_data1 = vld1q_u8(offset);
// Pairwise add and accumulate into accum vectors (16 bit to account
// for values above 255).
accum1 = vpadalq_u8(accum1, curr_data1);
}
// Second 16 bytes.
{
// Load 16 bytes of data from current offset.
const uint8x16_t curr_data2 = vld1q_u8(offset + 16);
// Pairwise add and accumulate into accum vectors (16 bit to account
// for values above 255).
accum2 = vpadalq_u8(accum2, curr_data2);
}
// Move offset down one row.
offset += stride;
}
// Divide by 4 (number of input pixels per output
// pixel) and narrow data from 16 bits per pixel to 8 bpp.
const uint8x8_t tmp_pix1 = vqshrn_n_u16(accum1, 2);
const uint8x8_t tmp_pix2 = vqshrn_n_u16(accum2, 2);
// Concatenate 8x1 pixel strips into 16x1 pixel strip.
const uint8x16_t allpixels = vcombine_u8(tmp_pix1, tmp_pix2);
// Copy all pixels from composite 16x1 vector into output strip.
vst1q_u8(ptr_dst, allpixels);
ptr_dst += stride_;
}
}
// This function does the bulk of the work.
template <>
void Image<uint8_t>::Downsample4x32ColumnsNeon(const uint8_t* const original,
const int stride,
const int orig_x) {
// Divide input x offset by 4 to find output offset.
const int new_x = orig_x >> 2;
// Initial offset into top row.
const uint8_t* offset = original + orig_x;
// This points to the leftmost pixel of our 8 horizontally arranged
// pixels in the destination data.
uint8_t* ptr_dst = (*this)[0] + new_x;
// Sum along vertical columns.
// Process 32x4 input pixels and 8x1 output pixels per iteration.
for (int new_y = 0; new_y < height_; ++new_y) {
uint16x8_t accum1 = vdupq_n_u16(0);
uint16x8_t accum2 = vdupq_n_u16(0);
// Go top to bottom across the four rows of input pixels that make up
// this output row.
for (int row_num = 0; row_num < 4; ++row_num) {
// First 16 bytes.
{
// Load 16 bytes of data from current offset.
const uint8x16_t curr_data1 = vld1q_u8(offset);
// Pairwise add and accumulate into accum vectors (16 bit to account
// for values above 255).
accum1 = vpadalq_u8(accum1, curr_data1);
}
// Second 16 bytes.
{
// Load 16 bytes of data from current offset.
const uint8x16_t curr_data2 = vld1q_u8(offset + 16);
// Pairwise add and accumulate into accum vectors (16 bit to account
// for values above 255).
accum2 = vpadalq_u8(accum2, curr_data2);
}
// Move offset down one row.
offset += stride;
}
// Add and widen, then divide by 16 (number of input pixels per output
// pixel) and narrow data from 32 bits per pixel to 16 bpp.
const uint16x4_t tmp_pix1 = vqshrn_n_u32(vpaddlq_u16(accum1), 4);
const uint16x4_t tmp_pix2 = vqshrn_n_u32(vpaddlq_u16(accum2), 4);
// Combine 4x1 pixel strips into 8x1 pixel strip and narrow from
// 16 bits to 8 bits per pixel.
const uint8x8_t allpixels = vmovn_u16(vcombine_u16(tmp_pix1, tmp_pix2));
// Copy all pixels from composite 8x1 vector into output strip.
vst1_u8(ptr_dst, allpixels);
ptr_dst += stride_;
}
}
// Hardware accelerated downsampling method for supported devices.
// Requires that image size be a multiple of 16 pixels in each dimension,
// and that downsampling be by a factor of 2 or 4.
template <>
void Image<uint8_t>::DownsampleAveragedNeon(const uint8_t* const original,
const int stride,
const int factor) {
// TODO(andrewharp): stride is a bad approximation for the src image's width.
// Better to pass that in directly.
SCHECK(width_ * factor <= stride, "Uh oh!");
const int last_starting_index = width_ * factor - 32;
// We process 32 input pixels lengthwise at a time.
// The output per pass of this loop is an 8 wide by downsampled height tall
// pixel strip.
int orig_x = 0;
for (; orig_x <= last_starting_index; orig_x += 32) {
if (factor == 2) {
Downsample2x32ColumnsNeon(original, stride, orig_x);
} else {
Downsample4x32ColumnsNeon(original, stride, orig_x);
}
}
// If a last pass is required, push it to the left enough so that it never
// goes out of bounds. This will result in some extra computation on devices
// whose frame widths are multiples of 16 and not 32.
if (orig_x < last_starting_index + 32) {
if (factor == 2) {
Downsample2x32ColumnsNeon(original, stride, last_starting_index);
} else {
Downsample4x32ColumnsNeon(original, stride, last_starting_index);
}
}
}
// Puts the image gradient matrix about a pixel into the 2x2 float array G.
// vals_x should be an array of the window x gradient values, whose indices
// can be in any order but are parallel to the vals_y entries.
// See http://robots.stanford.edu/cs223b04/algo_tracking.pdf for more details.
void CalculateGNeon(const float* const vals_x, const float* const vals_y,
const int num_vals, float* const G) {
const float32_t* const arm_vals_x = (const float32_t*) vals_x;
const float32_t* const arm_vals_y = (const float32_t*) vals_y;
// Running sums.
float32x4_t xx = vdupq_n_f32(0.0f);
float32x4_t xy = vdupq_n_f32(0.0f);
float32x4_t yy = vdupq_n_f32(0.0f);
// Maximum index we can load 4 consecutive values from.
// e.g. if there are 81 values, our last full pass can be from index 77:
// 81-4=>77 (77, 78, 79, 80)
const int max_i = num_vals - 4;
// Defined here because we want to keep track of how many values were
// processed by NEON, so that we can finish off the remainder the normal
// way.
int i = 0;
// Process values 4 at a time, accumulating the sums of
// the pixel-wise x*x, x*y, and y*y values.
for (; i <= max_i; i += 4) {
// Load xs
float32x4_t x = vld1q_f32(arm_vals_x + i);
// Multiply x*x and accumulate.
xx = vmlaq_f32(xx, x, x);
// Load ys
float32x4_t y = vld1q_f32(arm_vals_y + i);
// Multiply x*y and accumulate.
xy = vmlaq_f32(xy, x, y);
// Multiply y*y and accumulate.
yy = vmlaq_f32(yy, y, y);
}
static float32_t xx_vals[4];
static float32_t xy_vals[4];
static float32_t yy_vals[4];
vst1q_f32(xx_vals, xx);
vst1q_f32(xy_vals, xy);
vst1q_f32(yy_vals, yy);
// Accumulated values are store in sets of 4, we have to manually add
// the last bits together.
for (int j = 0; j < 4; ++j) {
G[0] += xx_vals[j];
G[1] += xy_vals[j];
G[3] += yy_vals[j];
}
// Finishes off last few values (< 4) from above.
for (; i < num_vals; ++i) {
G[0] += Square(vals_x[i]);
G[1] += vals_x[i] * vals_y[i];
G[3] += Square(vals_y[i]);
}
// The matrix is symmetric, so this is a given.
G[2] = G[1];
}
} // namespace tf_tracking
#endif
This diff is collapsed. Click to expand it.
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_INTEGRAL_IMAGE_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_INTEGRAL_IMAGE_H_
#include "tensorflow/examples/android/jni/object_tracking/geom.h"
#include "tensorflow/examples/android/jni/object_tracking/image-inl.h"
#include "tensorflow/examples/android/jni/object_tracking/image.h"
#include "tensorflow/examples/android/jni/object_tracking/utils.h"
namespace tf_tracking {
typedef uint8_t Code;
class IntegralImage : public Image<uint32_t> {
public:
explicit IntegralImage(const Image<uint8_t>& image_base)
: Image<uint32_t>(image_base.GetWidth(), image_base.GetHeight()) {
Recompute(image_base);
}
IntegralImage(const int width, const int height)
: Image<uint32_t>(width, height) {}
void Recompute(const Image<uint8_t>& image_base) {
SCHECK(image_base.GetWidth() == GetWidth() &&
image_base.GetHeight() == GetHeight(), "Dimensions don't match!");
// Sum along first row.
{
int x_sum = 0;
for (int x = 0; x < image_base.GetWidth(); ++x) {
x_sum += image_base[0][x];
(*this)[0][x] = x_sum;
}
}
// Sum everything else.
for (int y = 1; y < image_base.GetHeight(); ++y) {
uint32_t* curr_sum = (*this)[y];
// Previously summed pointers.
const uint32_t* up_one = (*this)[y - 1];
// Current value pointer.
const uint8_t* curr_delta = image_base[y];
uint32_t row_till_now = 0;
for (int x = 0; x < GetWidth(); ++x) {
// Add the one above and the one to the left.
row_till_now += *curr_delta;
*curr_sum = *up_one + row_till_now;
// Scoot everything along.
++curr_sum;
++up_one;
++curr_delta;
}
}
SCHECK(VerifyData(image_base), "Images did not match!");
}
bool VerifyData(const Image<uint8_t>& image_base) {
for (int y = 0; y < GetHeight(); ++y) {
for (int x = 0; x < GetWidth(); ++x) {
uint32_t curr_val = (*this)[y][x];
if (x > 0) {
curr_val -= (*this)[y][x - 1];
}
if (y > 0) {
curr_val -= (*this)[y - 1][x];
}
if (x > 0 && y > 0) {
curr_val += (*this)[y - 1][x - 1];
}
if (curr_val != image_base[y][x]) {
LOGE("Mismatch! %d vs %d", curr_val, image_base[y][x]);
return false;
}
if (GetRegionSum(x, y, x, y) != curr_val) {
LOGE("Mismatch!");
}
}
}
return true;
}
// Returns the sum of all pixels in the specified region.
inline uint32_t GetRegionSum(const int x1, const int y1, const int x2,
const int y2) const {
SCHECK(x1 >= 0 && y1 >= 0 &&
x2 >= x1 && y2 >= y1 && x2 < GetWidth() && y2 < GetHeight(),
"indices out of bounds! %d-%d / %d, %d-%d / %d, ",
x1, x2, GetWidth(), y1, y2, GetHeight());
const uint32_t everything = (*this)[y2][x2];
uint32_t sum = everything;
if (x1 > 0 && y1 > 0) {
// Most common case.
const uint32_t left = (*this)[y2][x1 - 1];
const uint32_t top = (*this)[y1 - 1][x2];
const uint32_t top_left = (*this)[y1 - 1][x1 - 1];
sum = everything - left - top + top_left;
SCHECK(sum >= 0, "Both: %d - %d - %d + %d => %d! indices: %d %d %d %d",
everything, left, top, top_left, sum, x1, y1, x2, y2);
} else if (x1 > 0) {
// Flush against top of image.
// Subtract out the region to the left only.
const uint32_t top = (*this)[y2][x1 - 1];
sum = everything - top;
SCHECK(sum >= 0, "Top: %d - %d => %d!", everything, top, sum);
} else if (y1 > 0) {
// Flush against left side of image.
// Subtract out the region above only.
const uint32_t left = (*this)[y1 - 1][x2];
sum = everything - left;
SCHECK(sum >= 0, "Left: %d - %d => %d!", everything, left, sum);
}
SCHECK(sum >= 0, "Negative sum!");
return sum;
}
// Returns the 2bit code associated with this region, which represents
// the overall gradient.
inline Code GetCode(const BoundingBox& bounding_box) const {
return GetCode(bounding_box.left_, bounding_box.top_,
bounding_box.right_, bounding_box.bottom_);
}
inline Code GetCode(const int x1, const int y1,
const int x2, const int y2) const {
SCHECK(x1 < x2 && y1 < y2, "Bounds out of order!! TL:%d,%d BR:%d,%d",
x1, y1, x2, y2);
// Gradient computed vertically.
const int box_height = (y2 - y1) / 2;
const int top_sum = GetRegionSum(x1, y1, x2, y1 + box_height);
const int bottom_sum = GetRegionSum(x1, y2 - box_height, x2, y2);
const bool vertical_code = top_sum > bottom_sum;
// Gradient computed horizontally.
const int box_width = (x2 - x1) / 2;
const int left_sum = GetRegionSum(x1, y1, x1 + box_width, y2);
const int right_sum = GetRegionSum(x2 - box_width, y1, x2, y2);
const bool horizontal_code = left_sum > right_sum;
const Code final_code = (vertical_code << 1) | horizontal_code;
SCHECK(InRange(final_code, static_cast<Code>(0), static_cast<Code>(3)),
"Invalid code! %d", final_code);
// Returns a value 0-3.
return final_code;
}
private:
TF_DISALLOW_COPY_AND_ASSIGN(IntegralImage);
};
} // namespace tf_tracking
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_INTEGRAL_IMAGE_H_
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_JNI_UTILS_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_JNI_UTILS_H_
#include <jni.h>
#include <stdint.h>
#include "tensorflow/examples/android/jni/object_tracking/utils.h"
// The JniLongField class is used to access Java fields from native code. This
// technique of hiding pointers to native objects in opaque Java fields is how
// the Android hardware libraries work. This reduces the amount of static
// native methods and makes it easier to manage the lifetime of native objects.
class JniLongField {
public:
JniLongField(const char* field_name)
: field_name_(field_name), field_ID_(0) {}
int64_t get(JNIEnv* env, jobject thiz) {
if (field_ID_ == 0) {
jclass cls = env->GetObjectClass(thiz);
CHECK_ALWAYS(cls != 0, "Unable to find class");
field_ID_ = env->GetFieldID(cls, field_name_, "J");
CHECK_ALWAYS(field_ID_ != 0,
"Unable to find field %s. (Check proguard cfg)", field_name_);
}
return env->GetLongField(thiz, field_ID_);
}
void set(JNIEnv* env, jobject thiz, int64_t value) {
if (field_ID_ == 0) {
jclass cls = env->GetObjectClass(thiz);
CHECK_ALWAYS(cls != 0, "Unable to find class");
field_ID_ = env->GetFieldID(cls, field_name_, "J");
CHECK_ALWAYS(field_ID_ != 0,
"Unable to find field %s (Check proguard cfg)", field_name_);
}
env->SetLongField(thiz, field_ID_, value);
}
private:
const char* const field_name_;
// This is just a cache
jfieldID field_ID_;
};
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_JNI_UTILS_H_
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_KEYPOINT_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_KEYPOINT_H_
#include "tensorflow/examples/android/jni/object_tracking/geom.h"
#include "tensorflow/examples/android/jni/object_tracking/image-inl.h"
#include "tensorflow/examples/android/jni/object_tracking/image.h"
#include "tensorflow/examples/android/jni/object_tracking/logging.h"
#include "tensorflow/examples/android/jni/object_tracking/time_log.h"
#include "tensorflow/examples/android/jni/object_tracking/utils.h"
#include "tensorflow/examples/android/jni/object_tracking/config.h"
namespace tf_tracking {
// For keeping track of keypoints.
struct Keypoint {
Keypoint() : pos_(0.0f, 0.0f), score_(0.0f), type_(0) {}
Keypoint(const float x, const float y)
: pos_(x, y), score_(0.0f), type_(0) {}
Point2f pos_;
float score_;
uint8_t type_;
};
inline std::ostream& operator<<(std::ostream& stream, const Keypoint keypoint) {
return stream << "[" << keypoint.pos_ << ", "
<< keypoint.score_ << ", " << keypoint.type_ << "]";
}
} // namespace tf_tracking
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_KEYPOINT_H_
This diff is collapsed. Click to expand it.
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_KEYPOINT_DETECTOR_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_KEYPOINT_DETECTOR_H_
#include <stdint.h>
#include <vector>
#include "tensorflow/examples/android/jni/object_tracking/image-inl.h"
#include "tensorflow/examples/android/jni/object_tracking/image.h"
#include "tensorflow/examples/android/jni/object_tracking/image_data.h"
#include "tensorflow/examples/android/jni/object_tracking/optical_flow.h"
namespace tf_tracking {
struct Keypoint;
class KeypointDetector {
public:
explicit KeypointDetector(const KeypointDetectorConfig* const config)
: config_(config),
keypoint_scratch_(new Image<uint8_t>(config_->image_size)),
interest_map_(new Image<bool>(config_->image_size)),
fast_quadrant_(0) {
interest_map_->Clear(false);
}
~KeypointDetector() {}
// Finds a new set of keypoints for the current frame, picked from the current
// set of keypoints and also from a set discovered via a keypoint detector.
// Special attention is applied to make sure that keypoints are distributed
// within the supplied ROIs.
void FindKeypoints(const ImageData& image_data,
const std::vector<BoundingBox>& rois,
const FramePair& prev_change,
FramePair* const curr_change);
private:
// Compute the corneriness of a point in the image.
float HarrisFilter(const Image<int32_t>& I_x, const Image<int32_t>& I_y,
const float x, const float y) const;
// Adds a grid of candidate keypoints to the given box, up to
// max_num_keypoints or kNumToAddAsCandidates^2, whichever is lower.
int AddExtraCandidatesForBoxes(
const std::vector<BoundingBox>& boxes,
const int max_num_keypoints,
Keypoint* const keypoints) const;
// Scan the frame for potential keypoints using the FAST keypoint detector.
// Quadrant is an argument 0-3 which refers to the quadrant of the image in
// which to detect keypoints.
int FindFastKeypoints(const Image<uint8_t>& frame, const int quadrant,
const int downsample_factor,
const int max_num_keypoints, Keypoint* const keypoints);
int FindFastKeypoints(const ImageData& image_data,
const int max_num_keypoints,
Keypoint* const keypoints);
// Score a bunch of candidate keypoints. Assigns the scores to the input
// candidate_keypoints array entries.
void ScoreKeypoints(const ImageData& image_data,
const int num_candidates,
Keypoint* const candidate_keypoints);
void SortKeypoints(const int num_candidates,
Keypoint* const candidate_keypoints) const;
// Selects a set of keypoints falling within the supplied box such that the
// most highly rated keypoints are picked first, and so that none of them are
// too close together.
int SelectKeypointsInBox(
const BoundingBox& box,
const Keypoint* const candidate_keypoints,
const int num_candidates,
const int max_keypoints,
const int num_existing_keypoints,
const Keypoint* const existing_keypoints,
Keypoint* const final_keypoints) const;
// Selects from the supplied sorted keypoint pool a set of keypoints that will
// best cover the given set of boxes, such that each box is covered at a
// resolution proportional to its size.
void SelectKeypoints(
const std::vector<BoundingBox>& boxes,
const Keypoint* const candidate_keypoints,
const int num_candidates,
FramePair* const frame_change) const;
// Copies and compacts the found keypoints in the second frame of prev_change
// into the array at new_keypoints.
static int CopyKeypoints(const FramePair& prev_change,
Keypoint* const new_keypoints);
const KeypointDetectorConfig* const config_;
// Scratch memory for keypoint candidacy detection and non-max suppression.
std::unique_ptr<Image<uint8_t> > keypoint_scratch_;
// Regions of the image to pay special attention to.
std::unique_ptr<Image<bool> > interest_map_;
// The current quadrant of the image to detect FAST keypoints in.
// Keypoint detection is staggered for performance reasons. Every four frames
// a full scan of the frame will have been performed.
int fast_quadrant_;
Keypoint tmp_keypoints_[kMaxTempKeypoints];
};
} // namespace tf_tracking
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_KEYPOINT_DETECTOR_H_
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/examples/android/jni/object_tracking/logging.h"
#ifdef STANDALONE_DEMO_LIB
#include <android/log.h>
#include <stdlib.h>
#include <time.h>
#include <iostream>
#include <sstream>
LogMessage::LogMessage(const char* fname, int line, int severity)
: fname_(fname), line_(line), severity_(severity) {}
void LogMessage::GenerateLogMessage() {
int android_log_level;
switch (severity_) {
case INFO:
android_log_level = ANDROID_LOG_INFO;
break;
case WARNING:
android_log_level = ANDROID_LOG_WARN;
break;
case ERROR:
android_log_level = ANDROID_LOG_ERROR;
break;
case FATAL:
android_log_level = ANDROID_LOG_FATAL;
break;
default:
if (severity_ < INFO) {
android_log_level = ANDROID_LOG_VERBOSE;
} else {
android_log_level = ANDROID_LOG_ERROR;
}
break;
}
std::stringstream ss;
const char* const partial_name = strrchr(fname_, '/');
ss << (partial_name != nullptr ? partial_name + 1 : fname_) << ":" << line_
<< " " << str();
__android_log_write(android_log_level, "native", ss.str().c_str());
// Also log to stderr (for standalone Android apps).
std::cerr << "native : " << ss.str() << std::endl;
// Android logging at level FATAL does not terminate execution, so abort()
// is still required to stop the program.
if (severity_ == FATAL) {
abort();
}
}
namespace {
// Parse log level (int64) from environment variable (char*)
int64_t LogLevelStrToInt(const char* tf_env_var_val) {
if (tf_env_var_val == nullptr) {
return 0;
}
// Ideally we would use env_var / safe_strto64, but it is
// hard to use here without pulling in a lot of dependencies,
// so we use std:istringstream instead
std::string min_log_level(tf_env_var_val);
std::istringstream ss(min_log_level);
int64_t level;
if (!(ss >> level)) {
// Invalid vlog level setting, set level to default (0)
level = 0;
}
return level;
}
int64_t MinLogLevelFromEnv() {
const char* tf_env_var_val = getenv("TF_CPP_MIN_LOG_LEVEL");
return LogLevelStrToInt(tf_env_var_val);
}
int64_t MinVLogLevelFromEnv() {
const char* tf_env_var_val = getenv("TF_CPP_MIN_VLOG_LEVEL");
return LogLevelStrToInt(tf_env_var_val);
}
} // namespace
LogMessage::~LogMessage() {
// Read the min log level once during the first call to logging.
static int64_t min_log_level = MinLogLevelFromEnv();
if (TF_PREDICT_TRUE(severity_ >= min_log_level)) GenerateLogMessage();
}
int64_t LogMessage::MinVLogLevel() {
static const int64_t min_vlog_level = MinVLogLevelFromEnv();
return min_vlog_level;
}
LogMessageFatal::LogMessageFatal(const char* file, int line)
: LogMessage(file, line, ANDROID_LOG_FATAL) {}
LogMessageFatal::~LogMessageFatal() {
// abort() ensures we don't return (we promised we would not via
// ATTRIBUTE_NORETURN).
GenerateLogMessage();
abort();
}
void LogString(const char* fname, int line, int severity,
const std::string& message) {
LogMessage(fname, line, severity) << message;
}
void LogPrintF(const int severity, const char* format, ...) {
char message[1024];
va_list argptr;
va_start(argptr, format);
vsnprintf(message, 1024, format, argptr);
va_end(argptr);
__android_log_write(severity, "native", message);
// Also log to stderr (for standalone Android apps).
std::cerr << "native : " << message << std::endl;
}
#endif
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_LOGGING_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_LOGGING_H_
#include <android/log.h>
#include <string.h>
#include <ostream>
#include <sstream>
#include <string>
// Allow this library to be built without depending on TensorFlow by
// defining STANDALONE_DEMO_LIB. Otherwise TensorFlow headers will be
// used.
#ifdef STANDALONE_DEMO_LIB
// A macro to disallow the copy constructor and operator= functions
// This is usually placed in the private: declarations for a class.
#define TF_DISALLOW_COPY_AND_ASSIGN(TypeName) \
TypeName(const TypeName&) = delete; \
void operator=(const TypeName&) = delete
#if defined(COMPILER_GCC3)
#define TF_PREDICT_FALSE(x) (__builtin_expect(x, 0))
#define TF_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
#else
#define TF_PREDICT_FALSE(x) (x)
#define TF_PREDICT_TRUE(x) (x)
#endif
// Log levels equivalent to those defined by
// third_party/tensorflow/core/platform/logging.h
const int INFO = 0; // base_logging::INFO;
const int WARNING = 1; // base_logging::WARNING;
const int ERROR = 2; // base_logging::ERROR;
const int FATAL = 3; // base_logging::FATAL;
const int NUM_SEVERITIES = 4; // base_logging::NUM_SEVERITIES;
class LogMessage : public std::basic_ostringstream<char> {
public:
LogMessage(const char* fname, int line, int severity);
~LogMessage();
// Returns the minimum log level for VLOG statements.
// E.g., if MinVLogLevel() is 2, then VLOG(2) statements will produce output,
// but VLOG(3) will not. Defaults to 0.
static int64_t MinVLogLevel();
protected:
void GenerateLogMessage();
private:
const char* fname_;
int line_;
int severity_;
};
// LogMessageFatal ensures the process will exit in failure after
// logging this message.
class LogMessageFatal : public LogMessage {
public:
LogMessageFatal(const char* file, int line);
~LogMessageFatal();
};
#define _TF_LOG_INFO \
::tensorflow::internal::LogMessage(__FILE__, __LINE__, tensorflow::INFO)
#define _TF_LOG_WARNING \
::tensorflow::internal::LogMessage(__FILE__, __LINE__, tensorflow::WARNING)
#define _TF_LOG_ERROR \
::tensorflow::internal::LogMessage(__FILE__, __LINE__, tensorflow::ERROR)
#define _TF_LOG_FATAL \
::tensorflow::internal::LogMessageFatal(__FILE__, __LINE__)
#define _TF_LOG_QFATAL _TF_LOG_FATAL
#define LOG(severity) _TF_LOG_##severity
#define VLOG_IS_ON(lvl) ((lvl) <= LogMessage::MinVLogLevel())
#define VLOG(lvl) \
if (TF_PREDICT_FALSE(VLOG_IS_ON(lvl))) \
LogMessage(__FILE__, __LINE__, ANDROID_LOG_INFO)
void LogPrintF(const int severity, const char* format, ...);
// Support for printf style logging.
#define LOGV(...)
#define LOGD(...)
#define LOGI(...) LogPrintF(ANDROID_LOG_INFO, __VA_ARGS__);
#define LOGW(...) LogPrintF(ANDROID_LOG_INFO, __VA_ARGS__);
#define LOGE(...) LogPrintF(ANDROID_LOG_ERROR, __VA_ARGS__);
#else
#include "tensorflow/core/lib/strings/stringprintf.h"
#include "tensorflow/core/platform/logging.h"
// Support for printf style logging.
#define LOGV(...)
#define LOGD(...)
#define LOGI(...) LOG(INFO) << tensorflow::strings::Printf(__VA_ARGS__);
#define LOGW(...) LOG(INFO) << tensorflow::strings::Printf(__VA_ARGS__);
#define LOGE(...) LOG(INFO) << tensorflow::strings::Printf(__VA_ARGS__);
#endif
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_LOGGING_H_
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// NOTE: no native object detectors are currently provided or used by the code
// in this directory. This class remains mainly for historical reasons.
// Detection in the TF demo is done through TensorFlowMultiBoxDetector.java.
#include "tensorflow/examples/android/jni/object_tracking/object_detector.h"
namespace tf_tracking {
// This is here so that the vtable gets created properly.
ObjectDetectorBase::~ObjectDetectorBase() {}
} // namespace tf_tracking
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// NOTE: no native object detectors are currently provided or used by the code
// in this directory. This class remains mainly for historical reasons.
// Detection in the TF demo is done through TensorFlowMultiBoxDetector.java.
// Defines the ObjectDetector class that is the main interface for detecting
// ObjectModelBases in frames.
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OBJECT_DETECTOR_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OBJECT_DETECTOR_H_
#include <float.h>
#include <map>
#include <memory>
#include <sstream>
#include <string>
#include <vector>
#include "tensorflow/examples/android/jni/object_tracking/geom.h"
#include "tensorflow/examples/android/jni/object_tracking/image-inl.h"
#include "tensorflow/examples/android/jni/object_tracking/image.h"
#include "tensorflow/examples/android/jni/object_tracking/integral_image.h"
#ifdef __RENDER_OPENGL__
#include "tensorflow/examples/android/jni/object_tracking/sprite.h"
#endif
#include "tensorflow/examples/android/jni/object_tracking/utils.h"
#include "tensorflow/examples/android/jni/object_tracking/config.h"
#include "tensorflow/examples/android/jni/object_tracking/image_data.h"
#include "tensorflow/examples/android/jni/object_tracking/object_model.h"
namespace tf_tracking {
// Adds BoundingSquares to a vector such that the first square added is centered
// in the position given and of square_size, and the remaining squares are added
// concentrentically, scaling down by scale_factor until the minimum threshold
// size is passed.
// Squares that do not fall completely within image_bounds will not be added.
static inline void FillWithSquares(
const BoundingBox& image_bounds,
const BoundingBox& position,
const float starting_square_size,
const float smallest_square_size,
const float scale_factor,
std::vector<BoundingSquare>* const squares) {
BoundingSquare descriptor_area =
GetCenteredSquare(position, starting_square_size);
SCHECK(scale_factor < 1.0f, "Scale factor too large at %.2f!", scale_factor);
// Use a do/while loop to ensure that at least one descriptor is created.
do {
if (image_bounds.Contains(descriptor_area.ToBoundingBox())) {
squares->push_back(descriptor_area);
}
descriptor_area.Scale(scale_factor);
} while (descriptor_area.size_ >= smallest_square_size - EPSILON);
LOGV("Created %zu squares starting from size %.2f to min size %.2f "
"using scale factor: %.2f",
squares->size(), starting_square_size, smallest_square_size,
scale_factor);
}
// Represents a potential detection of a specific ObjectExemplar and Descriptor
// at a specific position in the image.
class Detection {
public:
explicit Detection(const ObjectModelBase* const object_model,
const MatchScore match_score,
const BoundingBox& bounding_box)
: object_model_(object_model),
match_score_(match_score),
bounding_box_(bounding_box) {}
Detection(const Detection& other)
: object_model_(other.object_model_),
match_score_(other.match_score_),
bounding_box_(other.bounding_box_) {}
virtual ~Detection() {}
inline BoundingBox GetObjectBoundingBox() const {
return bounding_box_;
}
inline MatchScore GetMatchScore() const {
return match_score_;
}
inline const ObjectModelBase* GetObjectModel() const {
return object_model_;
}
inline bool Intersects(const Detection& other) {
// Check if any of the four axes separates us, there must be at least one.
return bounding_box_.Intersects(other.bounding_box_);
}
struct Comp {
inline bool operator()(const Detection& a, const Detection& b) const {
return a.match_score_ > b.match_score_;
}
};
// TODO(andrewharp): add accessors to update these instead.
const ObjectModelBase* object_model_;
MatchScore match_score_;
BoundingBox bounding_box_;
};
inline std::ostream& operator<<(std::ostream& stream,
const Detection& detection) {
const BoundingBox actual_area = detection.GetObjectBoundingBox();
stream << actual_area;
return stream;
}
class ObjectDetectorBase {
public:
explicit ObjectDetectorBase(const ObjectDetectorConfig* const config)
: config_(config),
image_data_(NULL) {}
virtual ~ObjectDetectorBase();
// Sets the current image data. All calls to ObjectDetector other than
// FillDescriptors use the image data last set.
inline void SetImageData(const ImageData* const image_data) {
image_data_ = image_data;
}
// Main entry point into the detection algorithm.
// Scans the frame for candidates, tweaks them, and fills in the
// given std::vector of Detection objects with acceptable matches.
virtual void Detect(const std::vector<BoundingSquare>& positions,
std::vector<Detection>* const detections) const = 0;
virtual ObjectModelBase* CreateObjectModel(const std::string& name) = 0;
virtual void DeleteObjectModel(const std::string& name) = 0;
virtual void GetObjectModels(
std::vector<const ObjectModelBase*>* models) const = 0;
// Creates a new ObjectExemplar from the given position in the context of
// the last frame passed to NextFrame.
// Will return null in the case that there's no room for a descriptor to be
// created in the example area, or the example area is not completely
// contained within the frame.
virtual void UpdateModel(const Image<uint8_t>& base_image,
const IntegralImage& integral_image,
const BoundingBox& bounding_box, const bool locked,
ObjectModelBase* model) const = 0;
virtual void Draw() const = 0;
virtual bool AllowSpontaneousDetections() = 0;
protected:
const std::unique_ptr<const ObjectDetectorConfig> config_;
// The latest frame data, upon which all detections will be performed.
// Not owned by this object, just provided for reference by ObjectTracker
// via SetImageData().
const ImageData* image_data_;
private:
TF_DISALLOW_COPY_AND_ASSIGN(ObjectDetectorBase);
};
template <typename ModelType>
class ObjectDetector : public ObjectDetectorBase {
public:
explicit ObjectDetector(const ObjectDetectorConfig* const config)
: ObjectDetectorBase(config) {}
virtual ~ObjectDetector() {
typename std::map<std::string, ModelType*>::const_iterator it =
object_models_.begin();
for (; it != object_models_.end(); ++it) {
ModelType* model = it->second;
delete model;
}
}
virtual void DeleteObjectModel(const std::string& name) {
ModelType* model = object_models_[name];
CHECK_ALWAYS(model != NULL, "Model was null!");
object_models_.erase(name);
SAFE_DELETE(model);
}
virtual void GetObjectModels(
std::vector<const ObjectModelBase*>* models) const {
typename std::map<std::string, ModelType*>::const_iterator it =
object_models_.begin();
for (; it != object_models_.end(); ++it) {
models->push_back(it->second);
}
}
virtual bool AllowSpontaneousDetections() {
return false;
}
protected:
std::map<std::string, ModelType*> object_models_;
private:
TF_DISALLOW_COPY_AND_ASSIGN(ObjectDetector);
};
} // namespace tf_tracking
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OBJECT_DETECTOR_H_
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// NOTE: no native object detectors are currently provided or used by the code
// in this directory. This class remains mainly for historical reasons.
// Detection in the TF demo is done through TensorFlowMultiBoxDetector.java.
// Contains ObjectModelBase declaration.
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OBJECT_MODEL_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OBJECT_MODEL_H_
#ifdef __RENDER_OPENGL__
#include <GLES/gl.h>
#include <GLES/glext.h>
#endif
#include <vector>
#include "tensorflow/examples/android/jni/object_tracking/geom.h"
#include "tensorflow/examples/android/jni/object_tracking/image-inl.h"
#include "tensorflow/examples/android/jni/object_tracking/image.h"
#include "tensorflow/examples/android/jni/object_tracking/integral_image.h"
#ifdef __RENDER_OPENGL__
#include "tensorflow/examples/android/jni/object_tracking/sprite.h"
#endif
#include "tensorflow/examples/android/jni/object_tracking/utils.h"
#include "tensorflow/examples/android/jni/object_tracking/config.h"
#include "tensorflow/examples/android/jni/object_tracking/image_data.h"
#include "tensorflow/examples/android/jni/object_tracking/keypoint.h"
namespace tf_tracking {
// The ObjectModelBase class represents all the known appearance information for
// an object. It is not a specific instance of the object in the world,
// but just the general appearance information that enables detection. An
// ObjectModelBase can be reused across multiple-instances of TrackedObjects.
class ObjectModelBase {
public:
ObjectModelBase(const std::string& name) : name_(name) {}
virtual ~ObjectModelBase() {}
// Called when the next step in an ongoing track occurs.
virtual void TrackStep(const BoundingBox& position,
const Image<uint8_t>& image,
const IntegralImage& integral_image,
const bool authoritative) {}
// Called when an object track is lost.
virtual void TrackLost() {}
// Called when an object track is confirmed as legitimate.
virtual void TrackConfirmed() {}
virtual float GetMaxCorrelation(const Image<float>& patch_image) const = 0;
virtual MatchScore GetMatchScore(
const BoundingBox& position, const ImageData& image_data) const = 0;
virtual void Draw(float* const depth) const = 0;
inline const std::string& GetName() const {
return name_;
}
protected:
const std::string name_;
private:
TF_DISALLOW_COPY_AND_ASSIGN(ObjectModelBase);
};
template <typename DetectorType>
class ObjectModel : public ObjectModelBase {
public:
ObjectModel<DetectorType>(const DetectorType* const detector,
const std::string& name)
: ObjectModelBase(name), detector_(detector) {}
protected:
const DetectorType* const detector_;
TF_DISALLOW_COPY_AND_ASSIGN(ObjectModel<DetectorType>);
};
} // namespace tf_tracking
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OBJECT_MODEL_H_
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OPTICAL_FLOW_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OPTICAL_FLOW_H_
#include "tensorflow/examples/android/jni/object_tracking/geom.h"
#include "tensorflow/examples/android/jni/object_tracking/image-inl.h"
#include "tensorflow/examples/android/jni/object_tracking/image.h"
#include "tensorflow/examples/android/jni/object_tracking/utils.h"
#include "tensorflow/examples/android/jni/object_tracking/config.h"
#include "tensorflow/examples/android/jni/object_tracking/frame_pair.h"
#include "tensorflow/examples/android/jni/object_tracking/image_data.h"
#include "tensorflow/examples/android/jni/object_tracking/keypoint.h"
namespace tf_tracking {
class FlowCache;
// Class encapsulating all the data and logic necessary for performing optical
// flow.
class OpticalFlow {
public:
explicit OpticalFlow(const OpticalFlowConfig* const config);
// Add a new frame to the optical flow. Will update all the non-keypoint
// related member variables.
//
// new_frame should be a buffer of grayscale values, one byte per pixel,
// at the original frame_width and frame_height used to initialize the
// OpticalFlow object. Downsampling will be handled internally.
//
// time_stamp should be a time in milliseconds that later calls to this and
// other methods will be relative to.
void NextFrame(const ImageData* const image_data);
// An implementation of the Lucas-Kanade Optical Flow algorithm.
static bool FindFlowAtPoint_LK(const Image<uint8_t>& img_I,
const Image<uint8_t>& img_J,
const Image<int32_t>& I_x,
const Image<int32_t>& I_y, const float p_x,
const float p_y, float* out_g_x,
float* out_g_y);
// Pointwise flow using translational 2dof ESM.
static bool FindFlowAtPoint_ESM(
const Image<uint8_t>& img_I, const Image<uint8_t>& img_J,
const Image<int32_t>& I_x, const Image<int32_t>& I_y,
const Image<int32_t>& J_x, const Image<int32_t>& J_y, const float p_x,
const float p_y, float* out_g_x, float* out_g_y);
// Finds the flow using a specific level, in either direction.
// If reversed, the coordinates are in the context of the latest
// frame, not the frame before it.
// All coordinates used in parameters are global, not scaled.
bool FindFlowAtPointReversible(
const int level, const float u_x, const float u_y,
const bool reverse_flow,
float* final_x, float* final_y) const;
// Finds the flow using a specific level, filterable by forward-backward
// error. All coordinates used in parameters are global, not scaled.
bool FindFlowAtPointSingleLevel(const int level,
const float u_x, const float u_y,
const bool filter_by_fb_error,
float* flow_x, float* flow_y) const;
// Pyramidal optical-flow using all levels.
bool FindFlowAtPointPyramidal(const float u_x, const float u_y,
const bool filter_by_fb_error,
float* flow_x, float* flow_y) const;
private:
const OpticalFlowConfig* const config_;
const ImageData* frame1_;
const ImageData* frame2_;
// Size of the internally allocated images (after original is downsampled).
const Size working_size_;
TF_DISALLOW_COPY_AND_ASSIGN(OpticalFlow);
};
} // namespace tf_tracking
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_OPTICAL_FLOW_H_
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_SPRITE_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_SPRITE_H_
#ifdef __RENDER_OPENGL__
#include <GLES/gl.h>
#include <GLES/glext.h>
#include "tensorflow/examples/android/jni/object_tracking/image-inl.h"
#include "tensorflow/examples/android/jni/object_tracking/image.h"
namespace tf_tracking {
// This class encapsulates the logic necessary to load an render image data
// at the same aspect ratio as the original source.
class Sprite {
public:
// Only create Sprites when you have an OpenGl context.
explicit Sprite(const Image<uint8_t>& image) { LoadTexture(image, NULL); }
Sprite(const Image<uint8_t>& image, const BoundingBox* const area) {
LoadTexture(image, area);
}
// Also, try to only delete a Sprite when holding an OpenGl context.
~Sprite() {
glDeleteTextures(1, &texture_);
}
inline int GetWidth() const {
return actual_width_;
}
inline int GetHeight() const {
return actual_height_;
}
// Draw the sprite at 0,0 - original width/height in the current reference
// frame. Any transformations desired must be applied before calling this
// function.
void Draw() const {
const float float_width = static_cast<float>(actual_width_);
const float float_height = static_cast<float>(actual_height_);
// Where it gets rendered to.
const float vertices[] = { 0.0f, 0.0f, 0.0f,
0.0f, float_height, 0.0f,
float_width, 0.0f, 0.0f,
float_width, float_height, 0.0f,
};
// The coordinates the texture gets drawn from.
const float max_x = float_width / texture_width_;
const float max_y = float_height / texture_height_;
const float textureVertices[] = {
0, 0,
0, max_y,
max_x, 0,
max_x, max_y,
};
glEnable(GL_TEXTURE_2D);
glBindTexture(GL_TEXTURE_2D, texture_);
glEnableClientState(GL_VERTEX_ARRAY);
glEnableClientState(GL_TEXTURE_COORD_ARRAY);
glVertexPointer(3, GL_FLOAT, 0, vertices);
glTexCoordPointer(2, GL_FLOAT, 0, textureVertices);
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
glDisableClientState(GL_VERTEX_ARRAY);
glDisableClientState(GL_TEXTURE_COORD_ARRAY);
}
private:
inline int GetNextPowerOfTwo(const int number) const {
int power_of_two = 1;
while (power_of_two < number) {
power_of_two *= 2;
}
return power_of_two;
}
// TODO(andrewharp): Allow sprites to have their textures reloaded.
void LoadTexture(const Image<uint8_t>& texture_source,
const BoundingBox* const area) {
glEnable(GL_TEXTURE_2D);
glGenTextures(1, &texture_);
glBindTexture(GL_TEXTURE_2D, texture_);
int left = 0;
int top = 0;
if (area != NULL) {
// If a sub-region was provided to pull the texture from, use that.
left = area->left_;
top = area->top_;
actual_width_ = area->GetWidth();
actual_height_ = area->GetHeight();
} else {
actual_width_ = texture_source.GetWidth();
actual_height_ = texture_source.GetHeight();
}
// The textures must be a power of two, so find the sizes that are large
// enough to contain the image data.
texture_width_ = GetNextPowerOfTwo(actual_width_);
texture_height_ = GetNextPowerOfTwo(actual_height_);
bool allocated_data = false;
uint8_t* texture_data;
// Except in the lucky case where we're not using a sub-region of the
// original image AND the source data has dimensions that are power of two,
// care must be taken to copy data at the appropriate source and destination
// strides so that the final block can be copied directly into texture
// memory.
// TODO(andrewharp): Figure out if data can be pulled directly from the
// source image with some alignment modifications.
if (left != 0 || top != 0 ||
actual_width_ != texture_source.GetWidth() ||
actual_height_ != texture_source.GetHeight()) {
texture_data = new uint8_t[actual_width_ * actual_height_];
for (int y = 0; y < actual_height_; ++y) {
memcpy(texture_data + actual_width_ * y, texture_source[top + y] + left,
actual_width_ * sizeof(uint8_t));
}
allocated_data = true;
} else {
// Cast away const-ness because for some reason glTexSubImage2D wants
// a non-const data pointer.
texture_data = const_cast<uint8_t*>(texture_source.data());
}
glTexImage2D(GL_TEXTURE_2D,
0,
GL_LUMINANCE,
texture_width_,
texture_height_,
0,
GL_LUMINANCE,
GL_UNSIGNED_BYTE,
NULL);
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
glTexSubImage2D(GL_TEXTURE_2D,
0,
0,
0,
actual_width_,
actual_height_,
GL_LUMINANCE,
GL_UNSIGNED_BYTE,
texture_data);
if (allocated_data) {
delete(texture_data);
}
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
}
// The id for the texture on the GPU.
GLuint texture_;
// The width and height to be used for display purposes, referring to the
// dimensions of the original texture.
int actual_width_;
int actual_height_;
// The allocated dimensions of the texture data, which must be powers of 2.
int texture_width_;
int texture_height_;
TF_DISALLOW_COPY_AND_ASSIGN(Sprite);
};
} // namespace tf_tracking
#endif // __RENDER_OPENGL__
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_SPRITE_H_
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/examples/android/jni/object_tracking/time_log.h"
#ifdef LOG_TIME
// Storage for logging functionality.
int num_time_logs = 0;
LogEntry time_logs[NUM_LOGS];
int num_avg_entries = 0;
AverageEntry avg_entries[NUM_LOGS];
#endif
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Utility functions for performance profiling.
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_TIME_LOG_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_TIME_LOG_H_
#include <stdint.h>
#include "tensorflow/examples/android/jni/object_tracking/logging.h"
#include "tensorflow/examples/android/jni/object_tracking/utils.h"
#ifdef LOG_TIME
// Blend constant for running average.
#define ALPHA 0.98f
#define NUM_LOGS 100
struct LogEntry {
const char* id;
int64_t time_stamp;
};
struct AverageEntry {
const char* id;
float average_duration;
};
// Storage for keeping track of this frame's values.
extern int num_time_logs;
extern LogEntry time_logs[NUM_LOGS];
// Storage for keeping track of average values (each entry may not be printed
// out each frame).
extern AverageEntry avg_entries[NUM_LOGS];
extern int num_avg_entries;
// Call this at the start of a logging phase.
inline static void ResetTimeLog() {
num_time_logs = 0;
}
// Log a message to be printed out when printTimeLog is called, along with the
// amount of time in ms that has passed since the last call to this function.
inline static void TimeLog(const char* const str) {
LOGV("%s", str);
if (num_time_logs >= NUM_LOGS) {
LOGE("Out of log entries!");
return;
}
time_logs[num_time_logs].id = str;
time_logs[num_time_logs].time_stamp = CurrentThreadTimeNanos();
++num_time_logs;
}
inline static float Blend(float old_val, float new_val) {
return ALPHA * old_val + (1.0f - ALPHA) * new_val;
}
inline static float UpdateAverage(const char* str, const float new_val) {
for (int entry_num = 0; entry_num < num_avg_entries; ++entry_num) {
AverageEntry* const entry = avg_entries + entry_num;
if (str == entry->id) {
entry->average_duration = Blend(entry->average_duration, new_val);
return entry->average_duration;
}
}
if (num_avg_entries >= NUM_LOGS) {
LOGE("Too many log entries!");
}
// If it wasn't there already, add it.
avg_entries[num_avg_entries].id = str;
avg_entries[num_avg_entries].average_duration = new_val;
++num_avg_entries;
return new_val;
}
// Prints out all the timeLog statements in chronological order with the
// interval that passed between subsequent statements. The total time between
// the first and last statements is printed last.
inline static void PrintTimeLog() {
LogEntry* last_time = time_logs;
float average_running_total = 0.0f;
for (int i = 0; i < num_time_logs; ++i) {
LogEntry* const this_time = time_logs + i;
const float curr_time =
(this_time->time_stamp - last_time->time_stamp) / 1000000.0f;
const float avg_time = UpdateAverage(this_time->id, curr_time);
average_running_total += avg_time;
LOGD("%32s: %6.3fms %6.4fms", this_time->id, curr_time, avg_time);
last_time = this_time;
}
const float total_time =
(last_time->time_stamp - time_logs->time_stamp) / 1000000.0f;
LOGD("TOTAL TIME: %6.3fms %6.4fms\n",
total_time, average_running_total);
LOGD(" ");
}
#else
inline static void ResetTimeLog() {}
inline static void TimeLog(const char* const str) {
LOGV("%s", str);
}
inline static void PrintTimeLog() {}
#endif
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_TIME_LOG_H_
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/examples/android/jni/object_tracking/tracked_object.h"
namespace tf_tracking {
static const float kInitialDistance = 20.0f;
static void InitNormalized(const Image<uint8_t>& src_image,
const BoundingBox& position,
Image<float>* const dst_image) {
BoundingBox scaled_box(position);
CopyArea(src_image, scaled_box, dst_image);
NormalizeImage(dst_image);
}
TrackedObject::TrackedObject(const std::string& id, const Image<uint8_t>& image,
const BoundingBox& bounding_box,
ObjectModelBase* const model)
: id_(id),
last_known_position_(bounding_box),
last_detection_position_(bounding_box),
position_last_computed_time_(-1),
object_model_(model),
last_detection_thumbnail_(kNormalizedThumbnailSize,
kNormalizedThumbnailSize),
last_frame_thumbnail_(kNormalizedThumbnailSize, kNormalizedThumbnailSize),
tracked_correlation_(0.0f),
tracked_match_score_(0.0),
num_consecutive_frames_below_threshold_(0),
allowable_detection_distance_(Square(kInitialDistance)) {
InitNormalized(image, bounding_box, &last_detection_thumbnail_);
}
TrackedObject::~TrackedObject() {}
void TrackedObject::UpdatePosition(const BoundingBox& new_position,
const int64_t timestamp,
const ImageData& image_data,
const bool authoritative) {
last_known_position_ = new_position;
position_last_computed_time_ = timestamp;
InitNormalized(*image_data.GetImage(), new_position, &last_frame_thumbnail_);
const float last_localization_correlation = ComputeCrossCorrelation(
last_detection_thumbnail_.data(),
last_frame_thumbnail_.data(),
last_frame_thumbnail_.data_size_);
LOGV("Tracked correlation to last localization: %.6f",
last_localization_correlation);
// Correlation to object model, if it exists.
if (object_model_ != NULL) {
tracked_correlation_ =
object_model_->GetMaxCorrelation(last_frame_thumbnail_);
LOGV("Tracked correlation to model: %.6f",
tracked_correlation_);
tracked_match_score_ =
object_model_->GetMatchScore(new_position, image_data);
LOGV("Tracked match score with model: %.6f",
tracked_match_score_.value);
} else {
// If there's no model to check against, set the tracked correlation to
// simply be the correlation to the last set position.
tracked_correlation_ = last_localization_correlation;
tracked_match_score_ = MatchScore(0.0f);
}
// Determine if it's still being tracked.
if (tracked_correlation_ >= kMinimumCorrelationForTracking &&
tracked_match_score_ >= kMinimumMatchScore) {
num_consecutive_frames_below_threshold_ = 0;
if (object_model_ != NULL) {
object_model_->TrackStep(last_known_position_, *image_data.GetImage(),
*image_data.GetIntegralImage(), authoritative);
}
} else if (tracked_match_score_ < kMatchScoreForImmediateTermination) {
if (num_consecutive_frames_below_threshold_ < 1000) {
LOGD("Tracked match score is way too low (%.6f), aborting track.",
tracked_match_score_.value);
}
// Add an absurd amount of missed frames so that all heuristics will
// consider it a lost track.
num_consecutive_frames_below_threshold_ += 1000;
if (object_model_ != NULL) {
object_model_->TrackLost();
}
} else {
++num_consecutive_frames_below_threshold_;
allowable_detection_distance_ *= 1.1f;
}
}
void TrackedObject::OnDetection(ObjectModelBase* const model,
const BoundingBox& detection_position,
const MatchScore match_score,
const int64_t timestamp,
const ImageData& image_data) {
const float overlap = detection_position.PascalScore(last_known_position_);
if (overlap > kPositionOverlapThreshold) {
// If the position agreement with the current tracked position is good
// enough, lock all the current unlocked examples.
object_model_->TrackConfirmed();
num_consecutive_frames_below_threshold_ = 0;
}
// Before relocalizing, make sure the new proposed position is better than
// the existing position by a small amount to prevent thrashing.
if (match_score <= tracked_match_score_ + kMatchScoreBuffer) {
LOGI("Not relocalizing since new match is worse: %.6f < %.6f + %.6f",
match_score.value, tracked_match_score_.value,
kMatchScoreBuffer.value);
return;
}
LOGI("Relocalizing! From (%.1f, %.1f)[%.1fx%.1f] to "
"(%.1f, %.1f)[%.1fx%.1f]: %.6f > %.6f",
last_known_position_.left_, last_known_position_.top_,
last_known_position_.GetWidth(), last_known_position_.GetHeight(),
detection_position.left_, detection_position.top_,
detection_position.GetWidth(), detection_position.GetHeight(),
match_score.value, tracked_match_score_.value);
if (overlap < kPositionOverlapThreshold) {
// The path might be good, it might be bad, but it's no longer a path
// since we're moving the box to a new position, so just nuke it from
// orbit to be safe.
object_model_->TrackLost();
}
object_model_ = model;
// Reset the last detected appearance.
InitNormalized(
*image_data.GetImage(), detection_position, &last_detection_thumbnail_);
num_consecutive_frames_below_threshold_ = 0;
last_detection_position_ = detection_position;
UpdatePosition(detection_position, timestamp, image_data, false);
allowable_detection_distance_ = Square(kInitialDistance);
}
} // namespace tf_tracking
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_TRACKED_OBJECT_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_TRACKED_OBJECT_H_
#ifdef __RENDER_OPENGL__
#include "tensorflow/examples/android/jni/object_tracking/gl_utils.h"
#endif
#include "tensorflow/examples/android/jni/object_tracking/object_detector.h"
namespace tf_tracking {
// A TrackedObject is a specific instance of an ObjectModel, with a known
// position in the world.
// It provides the last known position and number of recent detection failures,
// in addition to the more general appearance data associated with the object
// class (which is in ObjectModel).
// TODO(andrewharp): Make getters/setters follow styleguide.
class TrackedObject {
public:
TrackedObject(const std::string& id, const Image<uint8_t>& image,
const BoundingBox& bounding_box, ObjectModelBase* const model);
~TrackedObject();
void UpdatePosition(const BoundingBox& new_position, const int64_t timestamp,
const ImageData& image_data, const bool authoritative);
// This method is called when the tracked object is detected at a
// given position, and allows the associated Model to grow and/or prune
// itself based on where the detection occurred.
void OnDetection(ObjectModelBase* const model,
const BoundingBox& detection_position,
const MatchScore match_score, const int64_t timestamp,
const ImageData& image_data);
// Called when there's no detection of the tracked object. This will cause
// a tracking failure after enough consecutive failures if the area under
// the current bounding box also doesn't meet a minimum correlation threshold
// with the model.
void OnDetectionFailure() {}
inline bool IsVisible() const {
return tracked_correlation_ >= kMinimumCorrelationForTracking ||
num_consecutive_frames_below_threshold_ < kMaxNumDetectionFailures;
}
inline float GetCorrelation() {
return tracked_correlation_;
}
inline MatchScore GetMatchScore() {
return tracked_match_score_;
}
inline BoundingBox GetPosition() const {
return last_known_position_;
}
inline BoundingBox GetLastDetectionPosition() const {
return last_detection_position_;
}
inline const ObjectModelBase* GetModel() const {
return object_model_;
}
inline const std::string& GetName() const {
return id_;
}
inline void Draw() const {
#ifdef __RENDER_OPENGL__
if (tracked_correlation_ < kMinimumCorrelationForTracking) {
glColor4f(MAX(0.0f, -tracked_correlation_),
MAX(0.0f, tracked_correlation_),
0.0f,
1.0f);
} else {
glColor4f(MAX(0.0f, -tracked_correlation_),
MAX(0.0f, tracked_correlation_),
1.0f,
1.0f);
}
// Render the box itself.
BoundingBox temp_box(last_known_position_);
DrawBox(temp_box);
// Render a box inside this one (in case the actual box is hidden).
const float kBufferSize = 1.0f;
temp_box.left_ -= kBufferSize;
temp_box.top_ -= kBufferSize;
temp_box.right_ += kBufferSize;
temp_box.bottom_ += kBufferSize;
DrawBox(temp_box);
// Render one outside as well.
temp_box.left_ -= -2.0f * kBufferSize;
temp_box.top_ -= -2.0f * kBufferSize;
temp_box.right_ += -2.0f * kBufferSize;
temp_box.bottom_ += -2.0f * kBufferSize;
DrawBox(temp_box);
#endif
}
// Get current object's num_consecutive_frames_below_threshold_.
inline int64_t GetNumConsecutiveFramesBelowThreshold() {
return num_consecutive_frames_below_threshold_;
}
// Reset num_consecutive_frames_below_threshold_ to 0.
inline void resetNumConsecutiveFramesBelowThreshold() {
num_consecutive_frames_below_threshold_ = 0;
}
inline float GetAllowableDistanceSquared() const {
return allowable_detection_distance_;
}
private:
// The unique id used throughout the system to identify this
// tracked object.
const std::string id_;
// The last known position of the object.
BoundingBox last_known_position_;
// The last known position of the object.
BoundingBox last_detection_position_;
// When the position was last computed.
int64_t position_last_computed_time_;
// The object model this tracked object is representative of.
ObjectModelBase* object_model_;
Image<float> last_detection_thumbnail_;
Image<float> last_frame_thumbnail_;
// The correlation of the object model with the preview frame at its last
// tracked position.
float tracked_correlation_;
MatchScore tracked_match_score_;
// The number of consecutive frames that the tracked position for this object
// has been under the correlation threshold.
int num_consecutive_frames_below_threshold_;
float allowable_detection_distance_;
friend std::ostream& operator<<(std::ostream& stream,
const TrackedObject& tracked_object);
TF_DISALLOW_COPY_AND_ASSIGN(TrackedObject);
};
inline std::ostream& operator<<(std::ostream& stream,
const TrackedObject& tracked_object) {
stream << tracked_object.id_
<< " " << tracked_object.last_known_position_
<< " " << tracked_object.position_last_computed_time_
<< " " << tracked_object.num_consecutive_frames_below_threshold_
<< " " << tracked_object.object_model_
<< " " << tracked_object.tracked_correlation_;
return stream;
}
} // namespace tf_tracking
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_OBJECT_TRACKING_TRACKED_OBJECT_H_
This diff is collapsed. Click to expand it.
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// NEON implementations of Image methods for compatible devices. Control
// should never enter this compilation unit on incompatible devices.
#ifdef __ARM_NEON
#include <arm_neon.h>
#include "tensorflow/examples/android/jni/object_tracking/geom.h"
#include "tensorflow/examples/android/jni/object_tracking/image-inl.h"
#include "tensorflow/examples/android/jni/object_tracking/image.h"
#include "tensorflow/examples/android/jni/object_tracking/utils.h"
namespace tf_tracking {
inline static float GetSum(const float32x4_t& values) {
static float32_t summed_values[4];
vst1q_f32(summed_values, values);
return summed_values[0]
+ summed_values[1]
+ summed_values[2]
+ summed_values[3];
}
float ComputeMeanNeon(const float* const values, const int num_vals) {
SCHECK(num_vals >= 8, "Not enough values to merit NEON: %d", num_vals);
const float32_t* const arm_vals = (const float32_t* const) values;
float32x4_t accum = vdupq_n_f32(0.0f);
int offset = 0;
for (; offset <= num_vals - 4; offset += 4) {
accum = vaddq_f32(accum, vld1q_f32(&arm_vals[offset]));
}
// Pull the accumulated values into a single variable.
float sum = GetSum(accum);
// Get the remaining 1 to 3 values.
for (; offset < num_vals; ++offset) {
sum += values[offset];
}
const float mean_neon = sum / static_cast<float>(num_vals);
#ifdef SANITY_CHECKS
const float mean_cpu = ComputeMeanCpu(values, num_vals);
SCHECK(NearlyEqual(mean_neon, mean_cpu, EPSILON * num_vals),
"Neon mismatch with CPU mean! %.10f vs %.10f",
mean_neon, mean_cpu);
#endif
return mean_neon;
}
float ComputeStdDevNeon(const float* const values,
const int num_vals, const float mean) {
SCHECK(num_vals >= 8, "Not enough values to merit NEON: %d", num_vals);
const float32_t* const arm_vals = (const float32_t* const) values;
const float32x4_t mean_vec = vdupq_n_f32(-mean);
float32x4_t accum = vdupq_n_f32(0.0f);
int offset = 0;
for (; offset <= num_vals - 4; offset += 4) {
const float32x4_t deltas =
vaddq_f32(mean_vec, vld1q_f32(&arm_vals[offset]));
accum = vmlaq_f32(accum, deltas, deltas);
}
// Pull the accumulated values into a single variable.
float squared_sum = GetSum(accum);
// Get the remaining 1 to 3 values.
for (; offset < num_vals; ++offset) {
squared_sum += Square(values[offset] - mean);
}
const float std_dev_neon = sqrt(squared_sum / static_cast<float>(num_vals));
#ifdef SANITY_CHECKS
const float std_dev_cpu = ComputeStdDevCpu(values, num_vals, mean);
SCHECK(NearlyEqual(std_dev_neon, std_dev_cpu, EPSILON * num_vals),
"Neon mismatch with CPU std dev! %.10f vs %.10f",
std_dev_neon, std_dev_cpu);
#endif
return std_dev_neon;
}
float ComputeCrossCorrelationNeon(const float* const values1,
const float* const values2,
const int num_vals) {
SCHECK(num_vals >= 8, "Not enough values to merit NEON: %d", num_vals);
const float32_t* const arm_vals1 = (const float32_t* const) values1;
const float32_t* const arm_vals2 = (const float32_t* const) values2;
float32x4_t accum = vdupq_n_f32(0.0f);
int offset = 0;
for (; offset <= num_vals - 4; offset += 4) {
accum = vmlaq_f32(accum,
vld1q_f32(&arm_vals1[offset]),
vld1q_f32(&arm_vals2[offset]));
}
// Pull the accumulated values into a single variable.
float sxy = GetSum(accum);
// Get the remaining 1 to 3 values.
for (; offset < num_vals; ++offset) {
sxy += values1[offset] * values2[offset];
}
const float cross_correlation_neon = sxy / num_vals;
#ifdef SANITY_CHECKS
const float cross_correlation_cpu =
ComputeCrossCorrelationCpu(values1, values2, num_vals);
SCHECK(NearlyEqual(cross_correlation_neon, cross_correlation_cpu,
EPSILON * num_vals),
"Neon mismatch with CPU cross correlation! %.10f vs %.10f",
cross_correlation_neon, cross_correlation_cpu);
#endif
return cross_correlation_neon;
}
} // namespace tf_tracking
#endif // __ARM_NEON
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// These utility functions allow for the conversion of RGB data to YUV data.
#include "tensorflow/examples/android/jni/rgb2yuv.h"
static inline void WriteYUV(const int x, const int y, const int width,
const int r8, const int g8, const int b8,
uint8_t* const pY, uint8_t* const pUV) {
// Using formulas from http://msdn.microsoft.com/en-us/library/ms893078
*pY = ((66 * r8 + 129 * g8 + 25 * b8 + 128) >> 8) + 16;
// Odd widths get rounded up so that UV blocks on the side don't get cut off.
const int blocks_per_row = (width + 1) / 2;
// 2 bytes per UV block
const int offset = 2 * (((y / 2) * blocks_per_row + (x / 2)));
// U and V are the average values of all 4 pixels in the block.
if (!(x & 1) && !(y & 1)) {
// Explicitly clear the block if this is the first pixel in it.
pUV[offset] = 0;
pUV[offset + 1] = 0;
}
// V (with divide by 4 factored in)
#ifdef __APPLE__
const int u_offset = 0;
const int v_offset = 1;
#else
const int u_offset = 1;
const int v_offset = 0;
#endif
pUV[offset + v_offset] += ((112 * r8 - 94 * g8 - 18 * b8 + 128) >> 10) + 32;
// U (with divide by 4 factored in)
pUV[offset + u_offset] += ((-38 * r8 - 74 * g8 + 112 * b8 + 128) >> 10) + 32;
}
void ConvertARGB8888ToYUV420SP(const uint32_t* const input,
uint8_t* const output, int width, int height) {
uint8_t* pY = output;
uint8_t* pUV = output + (width * height);
const uint32_t* in = input;
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
const uint32_t rgb = *in++;
#ifdef __APPLE__
const int nB = (rgb >> 8) & 0xFF;
const int nG = (rgb >> 16) & 0xFF;
const int nR = (rgb >> 24) & 0xFF;
#else
const int nR = (rgb >> 16) & 0xFF;
const int nG = (rgb >> 8) & 0xFF;
const int nB = rgb & 0xFF;
#endif
WriteYUV(x, y, width, nR, nG, nB, pY++, pUV);
}
}
}
void ConvertRGB565ToYUV420SP(const uint16_t* const input, uint8_t* const output,
const int width, const int height) {
uint8_t* pY = output;
uint8_t* pUV = output + (width * height);
const uint16_t* in = input;
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
const uint32_t rgb = *in++;
const int r5 = ((rgb >> 11) & 0x1F);
const int g6 = ((rgb >> 5) & 0x3F);
const int b5 = (rgb & 0x1F);
// Shift left, then fill in the empty low bits with a copy of the high
// bits so we can stretch across the entire 0 - 255 range.
const int r8 = r5 << 3 | r5 >> 2;
const int g8 = g6 << 2 | g6 >> 4;
const int b8 = b5 << 3 | b5 >> 2;
WriteYUV(x, y, width, r8, g8, b8, pY++, pUV);
}
}
}
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_RGB2YUV_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_RGB2YUV_H_
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
void ConvertARGB8888ToYUV420SP(const uint32_t* const input,
uint8_t* const output, int width, int height);
void ConvertRGB565ToYUV420SP(const uint16_t* const input, uint8_t* const output,
const int width, const int height);
#ifdef __cplusplus
}
#endif
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_RGB2YUV_H_
VERS_1.0 {
# Export JNI symbols.
global:
Java_*;
JNI_OnLoad;
JNI_OnUnload;
# Hide everything else.
local:
*;
};
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This is a collection of routines which converts various YUV image formats
// to ARGB.
#include "tensorflow/examples/android/jni/yuv2rgb.h"
#ifndef MAX
#define MAX(a, b) ({__typeof__(a) _a = (a); __typeof__(b) _b = (b); _a > _b ? _a : _b; })
#define MIN(a, b) ({__typeof__(a) _a = (a); __typeof__(b) _b = (b); _a < _b ? _a : _b; })
#endif
// This value is 2 ^ 18 - 1, and is used to clamp the RGB values before their ranges
// are normalized to eight bits.
static const int kMaxChannelValue = 262143;
static inline uint32_t YUV2RGB(int nY, int nU, int nV) {
nY -= 16;
nU -= 128;
nV -= 128;
if (nY < 0) nY = 0;
// This is the floating point equivalent. We do the conversion in integer
// because some Android devices do not have floating point in hardware.
// nR = (int)(1.164 * nY + 2.018 * nU);
// nG = (int)(1.164 * nY - 0.813 * nV - 0.391 * nU);
// nB = (int)(1.164 * nY + 1.596 * nV);
int nR = 1192 * nY + 1634 * nV;
int nG = 1192 * nY - 833 * nV - 400 * nU;
int nB = 1192 * nY + 2066 * nU;
nR = MIN(kMaxChannelValue, MAX(0, nR));
nG = MIN(kMaxChannelValue, MAX(0, nG));
nB = MIN(kMaxChannelValue, MAX(0, nB));
nR = (nR >> 10) & 0xff;
nG = (nG >> 10) & 0xff;
nB = (nB >> 10) & 0xff;
return 0xff000000 | (nR << 16) | (nG << 8) | nB;
}
// Accepts a YUV 4:2:0 image with a plane of 8 bit Y samples followed by
// separate u and v planes with arbitrary row and column strides,
// containing 8 bit 2x2 subsampled chroma samples.
// Converts to a packed ARGB 32 bit output of the same pixel dimensions.
void ConvertYUV420ToARGB8888(const uint8_t* const yData,
const uint8_t* const uData,
const uint8_t* const vData, uint32_t* const output,
const int width, const int height,
const int y_row_stride, const int uv_row_stride,
const int uv_pixel_stride) {
uint32_t* out = output;
for (int y = 0; y < height; y++) {
const uint8_t* pY = yData + y_row_stride * y;
const int uv_row_start = uv_row_stride * (y >> 1);
const uint8_t* pU = uData + uv_row_start;
const uint8_t* pV = vData + uv_row_start;
for (int x = 0; x < width; x++) {
const int uv_offset = (x >> 1) * uv_pixel_stride;
*out++ = YUV2RGB(pY[x], pU[uv_offset], pV[uv_offset]);
}
}
}
// Accepts a YUV 4:2:0 image with a plane of 8 bit Y samples followed by an
// interleaved U/V plane containing 8 bit 2x2 subsampled chroma samples,
// except the interleave order of U and V is reversed. Converts to a packed
// ARGB 32 bit output of the same pixel dimensions.
void ConvertYUV420SPToARGB8888(const uint8_t* const yData,
const uint8_t* const uvData,
uint32_t* const output, const int width,
const int height) {
const uint8_t* pY = yData;
const uint8_t* pUV = uvData;
uint32_t* out = output;
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
int nY = *pY++;
int offset = (y >> 1) * width + 2 * (x >> 1);
#ifdef __APPLE__
int nU = pUV[offset];
int nV = pUV[offset + 1];
#else
int nV = pUV[offset];
int nU = pUV[offset + 1];
#endif
*out++ = YUV2RGB(nY, nU, nV);
}
}
}
// The same as above, but downsamples each dimension to half size.
void ConvertYUV420SPToARGB8888HalfSize(const uint8_t* const input,
uint32_t* const output, int width,
int height) {
const uint8_t* pY = input;
const uint8_t* pUV = input + (width * height);
uint32_t* out = output;
int stride = width;
width >>= 1;
height >>= 1;
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
int nY = (pY[0] + pY[1] + pY[stride] + pY[stride + 1]) >> 2;
pY += 2;
#ifdef __APPLE__
int nU = *pUV++;
int nV = *pUV++;
#else
int nV = *pUV++;
int nU = *pUV++;
#endif
*out++ = YUV2RGB(nY, nU, nV);
}
pY += stride;
}
}
// Accepts a YUV 4:2:0 image with a plane of 8 bit Y samples followed by an
// interleaved U/V plane containing 8 bit 2x2 subsampled chroma samples,
// except the interleave order of U and V is reversed. Converts to a packed
// RGB 565 bit output of the same pixel dimensions.
void ConvertYUV420SPToRGB565(const uint8_t* const input, uint16_t* const output,
const int width, const int height) {
const uint8_t* pY = input;
const uint8_t* pUV = input + (width * height);
uint16_t* out = output;
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
int nY = *pY++;
int offset = (y >> 1) * width + 2 * (x >> 1);
#ifdef __APPLE__
int nU = pUV[offset];
int nV = pUV[offset + 1];
#else
int nV = pUV[offset];
int nU = pUV[offset + 1];
#endif
nY -= 16;
nU -= 128;
nV -= 128;
if (nY < 0) nY = 0;
// This is the floating point equivalent. We do the conversion in integer
// because some Android devices do not have floating point in hardware.
// nR = (int)(1.164 * nY + 2.018 * nU);
// nG = (int)(1.164 * nY - 0.813 * nV - 0.391 * nU);
// nB = (int)(1.164 * nY + 1.596 * nV);
int nR = 1192 * nY + 1634 * nV;
int nG = 1192 * nY - 833 * nV - 400 * nU;
int nB = 1192 * nY + 2066 * nU;
nR = MIN(kMaxChannelValue, MAX(0, nR));
nG = MIN(kMaxChannelValue, MAX(0, nG));
nB = MIN(kMaxChannelValue, MAX(0, nB));
// Shift more than for ARGB8888 and apply appropriate bitmask.
nR = (nR >> 13) & 0x1f;
nG = (nG >> 12) & 0x3f;
nB = (nB >> 13) & 0x1f;
// R is high 5 bits, G is middle 6 bits, and B is low 5 bits.
*out++ = (nR << 11) | (nG << 5) | nB;
}
}
}
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This is a collection of routines which converts various YUV image formats
// to (A)RGB.
#ifndef TENSORFLOW_EXAMPLES_ANDROID_JNI_YUV2RGB_H_
#define TENSORFLOW_EXAMPLES_ANDROID_JNI_YUV2RGB_H_
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
void ConvertYUV420ToARGB8888(const uint8_t* const yData,
const uint8_t* const uData,
const uint8_t* const vData, uint32_t* const output,
const int width, const int height,
const int y_row_stride, const int uv_row_stride,
const int uv_pixel_stride);
// Converts YUV420 semi-planar data to ARGB 8888 data using the supplied width
// and height. The input and output must already be allocated and non-null.
// For efficiency, no error checking is performed.
void ConvertYUV420SPToARGB8888(const uint8_t* const pY,
const uint8_t* const pUV, uint32_t* const output,
const int width, const int height);
// The same as above, but downsamples each dimension to half size.
void ConvertYUV420SPToARGB8888HalfSize(const uint8_t* const input,
uint32_t* const output, int width,
int height);
// Converts YUV420 semi-planar data to RGB 565 data using the supplied width
// and height. The input and output must already be allocated and non-null.
// For efficiency, no error checking is performed.
void ConvertYUV420SPToRGB565(const uint8_t* const input, uint16_t* const output,
const int width, const int height);
#ifdef __cplusplus
}
#endif
#endif // TENSORFLOW_EXAMPLES_ANDROID_JNI_YUV2RGB_H_
<?xml version="1.0" encoding="utf-8"?><!--
Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<set xmlns:android="http://schemas.android.com/apk/res/android"
android:ordering="sequentially">
<objectAnimator
android:propertyName="backgroundColor"
android:duration="375"
android:valueFrom="0x00b3ccff"
android:valueTo="0xffb3ccff"
android:valueType="colorType"/>
<objectAnimator
android:propertyName="backgroundColor"
android:duration="375"
android:valueFrom="0xffb3ccff"
android:valueTo="0x00b3ccff"
android:valueType="colorType"/>
</set>
<?xml version="1.0" encoding="utf-8"?><!--
Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<shape xmlns:android="http://schemas.android.com/apk/res/android" android:shape="rectangle" >
<solid android:color="#00000000" />
<stroke android:width="1dip" android:color="#cccccc" />
</shape>
<?xml version="1.0" encoding="utf-8"?><!--
Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools"
android:id="@+id/container"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:background="#000"
tools:context="org.tensorflow.demo.CameraActivity" />
<?xml version="1.0" encoding="utf-8"?><!--
Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<FrameLayout
xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="match_parent"
tools:context="org.tensorflow.demo.SpeechActivity">
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="Say one of the words below!"
android:id="@+id/textView"
android:textAlignment="center"
android:layout_gravity="top"
android:textSize="24dp"
android:layout_marginTop="10dp"
android:layout_marginLeft="10dp"
/>
<ListView
android:id="@+id/list_view"
android:layout_width="240dp"
android:layout_height="wrap_content"
android:background="@drawable/border"
android:layout_gravity="top|center_horizontal"
android:textAlignment="center"
android:layout_marginTop="100dp"
/>
<Button
android:id="@+id/quit"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="Quit"
android:layout_gravity="bottom|center_horizontal"
android:layout_marginBottom="10dp"
/>
</FrameLayout>
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.