upload cursor texture in native plugin.

This commit is contained in:
hecomi
2016-11-20 22:35:52 +09:00
parent 49aef432e7
commit ddbfad6e95
13 changed files with 181 additions and 153 deletions
+2 -40
View File
@@ -1,6 +1,4 @@
using UnityEngine; using UnityEngine;
using UnityEngine.Assertions;
using System.Collections.Generic;
namespace uDesktopDuplication namespace uDesktopDuplication
{ {
@@ -16,18 +14,6 @@ public class Cursor : MonoBehaviour
private Texture uddTexture_; private Texture uddTexture_;
private Monitor monitor { get { return uddTexture_.monitor; } } private Monitor monitor { get { return uddTexture_.monitor; } }
struct TextureInfo
{
public Texture2D texture;
public System.IntPtr ptr;
public TextureInfo(Texture2D texture)
{
this.texture = texture;
this.ptr = texture.GetNativeTexturePtr();
}
}
private Dictionary<int, TextureInfo> textures_ = new Dictionary<int, TextureInfo>();
void Start() void Start()
{ {
uddTexture_ = GetComponent<Texture>(); uddTexture_ = GetComponent<Texture>();
@@ -37,9 +23,8 @@ public class Cursor : MonoBehaviour
{ {
if (monitor.isCursorVisible) { if (monitor.isCursorVisible) {
UpdatePosition(); UpdatePosition();
UpdateTexture();
} }
UpdateMaterial(); UpdateCoords();
} }
void UpdatePosition() void UpdatePosition()
@@ -51,33 +36,10 @@ public class Cursor : MonoBehaviour
worldPosition = transform.TransformPoint(localPos); worldPosition = transform.TransformPoint(localPos);
} }
void UpdateTexture() void UpdateCoords()
{
var w = monitor.cursorShapeWidth;
var h = monitor.cursorShapeHeight;
if (w == 0 || h == 0) return;
var key = w + h * 100;
if (!textures_.ContainsKey(key)) {
var texture = new Texture2D(w, h, TextureFormat.BGRA32, false);
texture.wrapMode = TextureWrapMode.Clamp;
textures_.Add(key, new TextureInfo(texture));
}
var cursorTexture = textures_[key];
Assert.IsNotNull(cursorTexture.texture);
monitor.GetCursorTexture(cursorTexture.ptr);
uddTexture_.material.SetTexture("_CursorTex", cursorTexture.texture);
}
void UpdateMaterial()
{ {
var x = monitor.isCursorVisible ? (float)monitor.cursorX / monitor.width : -9999f; var x = monitor.isCursorVisible ? (float)monitor.cursorX / monitor.width : -9999f;
var y = monitor.isCursorVisible ? (float)monitor.cursorY / monitor.height : -9999f; var y = monitor.isCursorVisible ? (float)monitor.cursorY / monitor.height : -9999f;
var w = (float)monitor.cursorShapeWidth / monitor.width;
var h = (float)monitor.cursorShapeHeight / monitor.height;
uddTexture_.material.SetVector("_CursorPositionScale", new Vector4(x, y, w, h));
coord = new Vector2(x, y); coord = new Vector2(x, y);
} }
} }
@@ -253,7 +253,7 @@ public class Monitor
public void Render() public void Render()
{ {
if (texture_ && available) { if (texture_ && available && texturePtr_ != System.IntPtr.Zero) {
Lib.SetTexturePtr(id, texturePtr_); Lib.SetTexturePtr(id, texturePtr_);
GL.IssuePluginEvent(Lib.GetRenderEventFunc(), id); GL.IssuePluginEvent(Lib.GetRenderEventFunc(), id);
} }
@@ -64,32 +64,14 @@ inline void uddConvertToLinearIfNeeded(inout fixed3 rgb)
} }
inline fixed4 uddGetScreenTexture(float2 uv) inline fixed4 uddGetScreenTexture(float2 uv)
{
fixed4 c = tex2D(_MainTex, uv);
return c;
}
inline fixed4 uddGetCursorTexture(float2 uv)
{
uv.x = (uv.x - _CursorX) / _CursorWidth;
uv.y = (uv.y - _CursorY) / _CursorHeight;
fixed4 c = tex2D(_CursorTex, uv);
fixed a = step(0, uv.x) * step(0, uv.y) * step(uv.x, 1) * step(uv.y, 1);
c.a *= step(0.01, a);
return c;
}
inline fixed4 uddGetScreenTextureWithCursor(float2 uv)
{ {
uv = uddInvertUV(uv); uv = uddInvertUV(uv);
#ifdef USE_CLIP #ifdef USE_CLIP
uv = uddClipUV(uv); uv = uddClipUV(uv);
#endif #endif
fixed4 screen = uddGetScreenTexture(uddRotateUV(uv)); fixed4 c = tex2D(_MainTex, uddRotateUV(uv));
fixed4 cursor = uddGetCursorTexture(uv); uddConvertToLinearIfNeeded(c.rgb);
fixed4 color = lerp(screen, cursor, cursor.a); return c;
uddConvertToLinearIfNeeded(color.rgb);
return color;
} }
inline void uddBendVertex(inout float4 v, half radius, half width, half thickness) inline void uddBendVertex(inout float4 v, half radius, half width, half thickness)
@@ -34,7 +34,7 @@ SubShader
void surf(Input IN, inout SurfaceOutputStandard o) void surf(Input IN, inout SurfaceOutputStandard o)
{ {
fixed4 c = uddGetScreenTextureWithCursor(IN.uv_MainTex) * _Color; fixed4 c = uddGetScreenTexture(IN.uv_MainTex) * _Color;
o.Albedo = c.rgb; o.Albedo = c.rgb;
o.Metallic = _Metallic; o.Metallic = _Metallic;
o.Smoothness = _Glossiness; o.Smoothness = _Glossiness;
@@ -39,7 +39,7 @@ v2f vert(appdata v)
fixed4 frag(v2f i) : SV_Target fixed4 frag(v2f i) : SV_Target
{ {
return uddGetScreenTextureWithCursor(i.uv); return uddGetScreenTexture(i.uv);
} }
ENDCG ENDCG
@@ -43,7 +43,7 @@ v2f vert(appdata v)
fixed4 frag(v2f i) : SV_Target fixed4 frag(v2f i) : SV_Target
{ {
fixed4 tex = uddGetScreenTextureWithCursor(i.uv); fixed4 tex = uddGetScreenTexture(i.uv);
fixed alpha = pow((tex.r + tex.g + tex.b) / 3.0, _Mask); fixed alpha = pow((tex.r + tex.g + tex.b) / 3.0, _Mask);
return fixed4(tex.rgb * _Color.rgb, alpha * _Color.a); return fixed4(tex.rgb * _Color.rgb, alpha * _Color.a);
} }
@@ -41,7 +41,7 @@ v2f vert(appdata v)
fixed4 frag(v2f i) : SV_Target fixed4 frag(v2f i) : SV_Target
{ {
return fixed4(uddGetScreenTextureWithCursor(i.uv).rgb, 1.0) * _Color; return fixed4(uddGetScreenTexture(i.uv).rgb, 1.0) * _Color;
} }
ENDCG ENDCG
@@ -83,7 +83,7 @@ public:
return value_ != nullptr; return value_ != nullptr;
} }
T operator [](UINT index) const const T operator [](UINT index) const
{ {
if (index >= size_) if (index >= size_)
{ {
@@ -93,6 +93,16 @@ public:
return value_[index]; return value_[index];
} }
T& operator [](UINT index)
{
if (index >= size_)
{
Debug::Error("Array index out of range: ", index, size_);
return value_[0];
}
return value_[index];
}
private: private:
std::unique_ptr<T[]> value_; std::unique_ptr<T[]> value_;
UINT size_ = 0; UINT size_ = 0;
@@ -79,33 +79,43 @@ void Cursor::UpdateTexture()
return; return;
} }
// Check desktop texure
if (monitor_->GetUnityTexture() == nullptr)
{
Debug::Error("Cursor::UpdateTexture() => Monitor::GetUnityTexture() is null.");
return;
}
// Cursor information // Cursor information
const bool isMono = GetType() == DXGI_OUTDUPL_POINTER_SHAPE_TYPE_MONOCHROME;
const bool isColorMask = GetType() == DXGI_OUTDUPL_POINTER_SHAPE_TYPE_MASKED_COLOR;
const auto cursorImageWidth = GetWidth(); const auto cursorImageWidth = GetWidth();
const auto cursorImageHeight = GetHeight(); const auto cursorImageHeight = GetHeight();
const auto cursorImagePitch = GetPitch(); const auto cursorImagePitch = GetPitch();
// Captured area // Captured size
auto capturedImageWidth = cursorImageWidth; auto capturedImageWidth = cursorImageWidth;
auto capturedImageHeight = cursorImageHeight; auto capturedImageHeight = cursorImageHeight;
// Convert the buffer given by API into BGRA32 // Convert the buffer given by API into BGRA32
const UINT bgraBufferSize = cursorImageWidth * cursorImageHeight * 4; const UINT bgraBufferSize = cursorImageWidth * cursorImageHeight * 4;
bgra32Buffer_.ExpandIfNeeded(bgraBufferSize); bgraBuffer_.ExpandIfNeeded(bgraBufferSize);
// Check buffers // Check buffers
if (!bgra32Buffer_ || !apiBuffer_) if (!bgraBuffer_ || !apiBuffer_)
{ {
return; return;
} }
// If masked, copy the desktop image and merge it with masked image. // Calculate information to capture desktop image under cursor.
if (isMono || isColorMask) D3D11_BOX box;
const auto monitorRot = static_cast<DXGI_MODE_ROTATION>(monitor_->GetRotation());
auto colMin = 0;
auto colMax = cursorImageWidth;
auto rowMin = 0;
auto rowMax = cursorImageHeight;
{ {
const auto monitorWidth = monitor_->GetWidth(); const auto monitorWidth = monitor_->GetWidth();
const auto monitorHeight = monitor_->GetHeight(); const auto monitorHeight = monitor_->GetHeight();
const auto monitorRot = static_cast<DXGI_MODE_ROTATION>(monitor_->GetRotation());
const auto isVertical = const auto isVertical =
monitorRot == DXGI_MODE_ROTATION_ROTATE90 || monitorRot == DXGI_MODE_ROTATION_ROTATE90 ||
monitorRot == DXGI_MODE_ROTATION_ROTATE270; monitorRot == DXGI_MODE_ROTATION_ROTATE270;
@@ -114,10 +124,6 @@ void Cursor::UpdateTexture()
auto x = x_; auto x = x_;
auto y = y_; auto y = y_;
auto colMin = 0;
auto colMax = cursorImageWidth;
auto rowMin = 0;
auto rowMax = cursorImageHeight;
if (x < 0) if (x < 0)
{ {
@@ -142,7 +148,6 @@ void Cursor::UpdateTexture()
rowMax = capturedImageHeight; rowMax = capturedImageHeight;
} }
D3D11_BOX box;
box.front = 0; box.front = 0;
box.back = 1; box.back = 1;
@@ -196,13 +201,11 @@ void Cursor::UpdateTexture()
"(", desktopImageWidth, ", ", desktopImageHeight, ")"); "(", desktopImageWidth, ", ", desktopImageHeight, ")");
return; return;
} }
}
if (monitor_->GetUnityTexture() == nullptr) // Create texture
{ ComPtr<ID3D11Texture2D> texture;
Debug::Error("Cursor::UpdateTexture() => Monitor::GetUnityTexture() is null."); {
return;
}
D3D11_TEXTURE2D_DESC desc; D3D11_TEXTURE2D_DESC desc;
desc.Width = capturedImageWidth; desc.Width = capturedImageWidth;
desc.Height = capturedImageHeight; desc.Height = capturedImageHeight;
@@ -216,79 +219,85 @@ void Cursor::UpdateTexture()
desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
desc.MiscFlags = 0; desc.MiscFlags = 0;
ComPtr<ID3D11Texture2D> texture;
if (FAILED(GetDevice()->CreateTexture2D(&desc, nullptr, &texture))) if (FAILED(GetDevice()->CreateTexture2D(&desc, nullptr, &texture)))
{ {
Debug::Error("Cursor::UpdateTexture() => GetDevice()->CreateTexture2D() failed."); Debug::Error("Cursor::UpdateTexture() => GetDevice()->CreateTexture2D() failed.");
return; return;
} }
}
// Copy desktop image to the texture
{
ComPtr<ID3D11DeviceContext> context;
GetDevice()->GetImmediateContext(&context);
context->CopySubresourceRegion(texture.Get(), 0, 0, 0, 0, monitor_->GetUnityTexture(), 0, &box);
}
// Get mapped surface
ComPtr<IDXGISurface> surface;
if (FAILED(texture.As(&surface)))
{
Debug::Error("Cursor::UpdateTexture() => texture->QueryInterface() failed.");
return;
}
DXGI_MAPPED_RECT mappedSurface;
if (FAILED(surface->Map(&mappedSurface, DXGI_MAP_READ)))
{
Debug::Error("Cursor::UpdateTexture() => surface->Map() failed.");
return;
}
// Finally, get the desktop texture under the mouse cursor.
const auto desktop32 = reinterpret_cast<UINT*>(mappedSurface.pBits);
const UINT desktopPitch = mappedSurface.Pitch / sizeof(UINT);
// Take the monitor orientation into consideration.
const auto getDesktop32 = [&](int col, int row)
{
switch (monitorRot)
{ {
ComPtr<ID3D11DeviceContext> context; case DXGI_MODE_ROTATION_ROTATE90:
GetDevice()->GetImmediateContext(&context); return desktop32[(capturedImageWidth - 1 - col) * desktopPitch + row];
context->CopySubresourceRegion(texture.Get(), 0, 0, 0, 0, monitor_->GetUnityTexture(), 0, &box); case DXGI_MODE_ROTATION_ROTATE180:
return desktop32[(capturedImageHeight - 1 - row) * desktopPitch + (capturedImageWidth - 1 - col)];
case DXGI_MODE_ROTATION_ROTATE270:
return desktop32[col * desktopPitch + (capturedImageHeight - 1 - row)];
case DXGI_MODE_ROTATION_IDENTITY:
case DXGI_MODE_ROTATION_UNSPECIFIED:
return desktop32[row * desktopPitch + col];
} }
};
ComPtr<IDXGISurface> surface; // Access RGBA values at the same time
if (FAILED(texture.As(&surface))) Buffer<BYTE> output;
{ output.ExpandIfNeeded(bgraBuffer_.Size());
Debug::Error("Cursor::UpdateTexture() => texture->QueryInterface() failed."); auto output32 = output.As<UINT>();
return;
}
DXGI_MAPPED_RECT mappedSurface; switch (GetType())
if (FAILED(surface->Map(&mappedSurface, DXGI_MAP_READ))) {
{ case DXGI_OUTDUPL_POINTER_SHAPE_TYPE_MONOCHROME:
Debug::Error("Cursor::UpdateTexture() => surface->Map() failed.");
return;
}
// Finally, get the texture behind the mouse cursor.
const auto desktop32 = reinterpret_cast<UINT*>(mappedSurface.pBits);
const UINT desktopPitch = mappedSurface.Pitch / sizeof(UINT);
// Take the monitor orientation into consideration.
const auto getDesktop32 = [&](int col, int row)
{
switch (monitorRot)
{
case DXGI_MODE_ROTATION_ROTATE90:
return desktop32[(capturedImageWidth - 1 - col) * desktopPitch + row];
break;
case DXGI_MODE_ROTATION_ROTATE180:
return desktop32[(capturedImageHeight - 1 - row) * desktopPitch + (capturedImageWidth - 1 - col)];
break;
case DXGI_MODE_ROTATION_ROTATE270:
return desktop32[col * desktopPitch + (capturedImageHeight - 1 - row)];
break;
case DXGI_MODE_ROTATION_IDENTITY:
case DXGI_MODE_ROTATION_UNSPECIFIED:
return desktop32[row * desktopPitch + col];
break;
}
};
// Access RGBA values at the same time
auto output32 = reinterpret_cast<UINT*>(bgra32Buffer_.Get());
if (isMono)
{ {
for (int row = rowMin, y = 0; row < rowMax; ++row, ++y) for (int row = rowMin, y = 0; row < rowMax; ++row, ++y)
{ {
for (int col = colMin, x = 0; col < colMax; ++col, ++x) for (int col = colMin, x = 0; col < colMax; ++col, ++x)
{ {
const int i = col + row * cursorImageWidth;
BYTE mask = 0b10000000 >> (col % 8); BYTE mask = 0b10000000 >> (col % 8);
const BYTE andMask = apiBuffer_[col / 8 + row * cursorImagePitch] & mask; const BYTE andMask = apiBuffer_[col / 8 + row * cursorImagePitch] & mask;
const BYTE xorMask = apiBuffer_[col / 8 + (row + capturedImageHeight) * cursorImagePitch] & mask; const BYTE xorMask = apiBuffer_[col / 8 + (row + capturedImageHeight) * cursorImagePitch] & mask;
const UINT andMask32 = andMask ? 0xFFFFFFFF : 0x00000000; const UINT andMask32 = andMask ? 0xFFFFFFFF : 0x00000000;
const UINT xorMask32 = xorMask ? 0xFFFFFFFF : 0x00000000; const UINT xorMask32 = xorMask ? 0xFFFFFFFF : 0x00000000;
output32[row * cursorImageWidth + col] = (getDesktop32(x, y) & andMask32) ^ xorMask32;
output32[i] = (getDesktop32(x, y) & andMask32) ^ xorMask32;
} }
} }
break;
} }
else // DXGI_OUTDUPL_POINTER_SHAPE_TYPE_MASKED_COLOR case DXGI_OUTDUPL_POINTER_SHAPE_TYPE_MASKED_COLOR:
{ {
const auto buffer32 = reinterpret_cast<UINT*>(apiBuffer_.Get()); const auto buffer32 = apiBuffer_.As<UINT>();
for (int row = rowMin, y = 0; row < rowMax; ++row, ++y) for (int row = rowMin, y = 0; row < rowMax; ++row, ++y)
{ {
@@ -308,28 +317,92 @@ void Cursor::UpdateTexture()
} }
} }
} }
break;
} }
case DXGI_OUTDUPL_POINTER_SHAPE_TYPE_COLOR:
if (FAILED(surface->Unmap()))
{ {
return; const auto buffer32 = apiBuffer_.As<UINT>();
auto output32 = bgraBuffer_.As<UINT>();
for (int row = rowMin, y = 0; row < rowMax; ++row, ++y)
{
for (int col = colMin, x = 0; col < colMax; ++col, ++x)
{
const int i = 4 * (col + row * cursorImageWidth);
const auto desktop32 = getDesktop32(x, y);
const auto desktop = (BYTE*)(&desktop32);
const auto desktopB = desktop[0];
const auto desktopG = desktop[1];
const auto desktopR = desktop[2];
const auto desktopA = desktop[3];
const auto cursorB = apiBuffer_[i + 0];
const auto cursorG = apiBuffer_[i + 1];
const auto cursorR = apiBuffer_[i + 2];
const auto cursorA = apiBuffer_[i + 3];
const auto a0 = cursorA / 255.f;
const auto a1 = 1.f - a0;
output[i + 0] = static_cast<BYTE>(cursorB * a0 + desktopB * a1);
output[i + 1] = static_cast<BYTE>(cursorG * a0 + desktopG * a1);
output[i + 2] = static_cast<BYTE>(cursorR * a0 + desktopR * a1);
output[i + 3] = desktopA;
}
}
break;
}
default:
{
Debug::Error("Cursor::UpdateTexture() => Unknown cursor type");
break;
} }
} }
else // DXGI_OUTDUPL_POINTER_SHAPE_TYPE_COLOR
// Rotation
auto bgraBuffer32 = bgraBuffer_.As<UINT>();
for (int row = rowMin, y = 0; row < rowMax; ++row, ++y)
{ {
auto output32 = reinterpret_cast<UINT*>(bgra32Buffer_.Get()); for (int col = colMin, x = 0; col < colMax; ++col, ++x)
const auto buffer32 = reinterpret_cast<UINT*>(apiBuffer_.Get());
for (int i = 0; i < cursorImageWidth * cursorImageHeight; ++i)
{ {
output32[i] = buffer32[i]; const auto i = col + row * cursorImageWidth;
switch (monitorRot)
{
case DXGI_MODE_ROTATION_ROTATE90:
bgraBuffer32[i] = output32[col * cursorImageHeight + (cursorImageHeight - 1 - row)];
break;
case DXGI_MODE_ROTATION_ROTATE180:
bgraBuffer32[i] = output32[(cursorImageHeight - 1 - row) * cursorImageWidth + (cursorImageWidth - 1 - col)];
break;
case DXGI_MODE_ROTATION_ROTATE270:
bgraBuffer32[i] = output32[(cursorImageWidth - 1 - col) * cursorImageHeight + row];
break;
case DXGI_MODE_ROTATION_IDENTITY:
case DXGI_MODE_ROTATION_UNSPECIFIED:
bgraBuffer32[i] = output32[i];
break;
}
} }
} }
{
ComPtr<ID3D11DeviceContext> context;
GetDevice()->GetImmediateContext(&context);
context->UpdateSubresource(monitor_->GetUnityTexture(), 0, &box, bgraBuffer_.Get(), GetWidth() * 4, 0);
}
if (FAILED(surface->Unmap()))
{
Debug::Error("Cursor::UpdateTexture() => surface->Unmap() failed.");
return;
}
} }
void Cursor::GetTexture(ID3D11Texture2D* texture) void Cursor::GetTexture(ID3D11Texture2D* texture)
{ {
if (!bgra32Buffer_) if (!bgraBuffer_)
{ {
Debug::Error("Cursor::GetTexture() => bgra32Buffer is null."); Debug::Error("Cursor::GetTexture() => bgra32Buffer is null.");
return; return;
@@ -356,7 +429,7 @@ void Cursor::GetTexture(ID3D11Texture2D* texture)
ComPtr<ID3D11DeviceContext> context; ComPtr<ID3D11DeviceContext> context;
GetDevice()->GetImmediateContext(&context); GetDevice()->GetImmediateContext(&context);
context->UpdateSubresource(texture, 0, nullptr, bgra32Buffer_.Get(), GetWidth() * 4, 0); context->UpdateSubresource(texture, 0, nullptr, bgraBuffer_.Get(), GetWidth() * 4, 0);
} }
@@ -32,7 +32,7 @@ private:
int x_ = -1; int x_ = -1;
int y_ = -1; int y_ = -1;
Buffer<BYTE> apiBuffer_; Buffer<BYTE> apiBuffer_;
Buffer<BYTE> bgra32Buffer_; Buffer<BYTE> bgraBuffer_;
DXGI_OUTDUPL_POINTER_SHAPE_INFO shapeInfo_; DXGI_OUTDUPL_POINTER_SHAPE_INFO shapeInfo_;
LARGE_INTEGER timestamp_; LARGE_INTEGER timestamp_;
}; };
@@ -112,10 +112,11 @@
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile> <ClCompile>
<WarningLevel>Level3</WarningLevel> <WarningLevel>Level3</WarningLevel>
<Optimization>MaxSpeed</Optimization> <Optimization>Full</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking> <FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions> <IntrinsicFunctions>true</IntrinsicFunctions>
<SDLCheck>true</SDLCheck> <SDLCheck>true</SDLCheck>
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
</ClCompile> </ClCompile>
<Link> <Link>
<EnableCOMDATFolding>true</EnableCOMDATFolding> <EnableCOMDATFolding>true</EnableCOMDATFolding>