Select * from MyTable
where CONVERT(VARBINARY(4), SUBSTRING(foo, 10, 4), 2) &10= 10

Path<String> field = root.get(MyTable_.foo);
Expression<String> expression = cb.substring(field, 10, 4);

ParameterExpression<?> param1 = cb.parameter(String.class, "convertParam1");
cb.function("CONVERT", Byte[].class, param1, expression);
query.setParameter("convertParam1", "VARBINARY");

CONVERT('VARBINARY', SUBSTRING(foo, 10, 4))

CONVERT(VARBINARY, SUBSTRING(foo, 10, 4))

ALTER FUNCTION [dbo].[testIt](@foo varchar(50), @starting_position int, @length int, @bit_expression int)
returns int
as
begin
   return CONVERT(VARBINARY(4), SUBSTRING(@foo, @starting_position, @length), 2) &@bit_expression;
end

public List<MyTable> test() {
   int startPos = 10;
   int length = 4;
   int bitwise = 10;
      
   // criteria sql
   CriteriaBuilder cb = em.getCriteriaBuilder();
   CriteriaQuery<MyTable> cq = cb.createQuery(MyTable.class);

   Root<MyTable> root = cq.from(MyTable.class);
   cq.select(root);
      
   Path<String> field = root.get(MyTable_.foo);
      
   Expression<?> startPosExpr = cb.literal(startPos);
   Expression<?> lengthExpr = cb.literal(length);
   Expression<?> bitwiseExpr = cb.literal(bitwise);
      
   Expression<?> functExpr= cb.function("dbo.testIt", Integer[].class, field, startPosExpr, lengthExpr, bitwiseExpr);
      
   Predicate binaryConditionPred = cb.equal(functExpr, bitwise);
      
   cq.where(binaryConditionPred);

   List<MyTable> entries;
   try {
      TypedQuery<MyTable> query = em.createQuery(cq).setMaxResults(100);
      entries = query.getResultList();
   } catch (NoResultException e) {
      entries = new ArrayList<MyTable>();
   }
   return entries;
}